From 9a36783132e2ed930c21186cad6b475ee7917224 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 8 Jul 2026 11:39:16 +0100 Subject: [PATCH 0001/1072] feat: Support RHOAI RC images Signed-off-by: Alberto Perdomo --- .../toolbox/deploy_custom_catalog/main.py | 3 + .../templates/catalogsource.yaml.j2 | 3 + projects/llm_d/orchestration/ci.py | 29 +++- .../orchestration/config.d/platform.yaml | 8 +- .../manifests/quay-registry-icsp.yaml | 9 ++ projects/llm_d/orchestration/prepare_phase.py | 134 ++++++++++++++++++ projects/llm_d/tests/test_profiles.py | 110 +++++++++++++- vaults/psap-rhoai-rc.yaml | 8 ++ 8 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml create mode 100644 vaults/psap-rhoai-rc.yaml diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index eea38593d..641184cb8 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -17,6 +17,7 @@ def run( catalog_image: str, *, display_name: str = "", + publisher: str = "", ) -> int: """ Deploy a CatalogSource from a custom index image and wait for it to become READY. @@ -26,6 +27,7 @@ def run( catalog_namespace: Namespace where the CatalogSource will be deployed catalog_image: Index image backing the CatalogSource display_name: Optional human-friendly label for logs + publisher: Optional publisher label for the CatalogSource """ execute_tasks(locals()) @@ -39,6 +41,7 @@ def setup_directories(args, ctx): shell.mkdir("src") shell.mkdir("artifacts") ctx.display_name = args.display_name or args.catalog_source_name + ctx.publisher = args.publisher or "" return f"Prepared directories for {ctx.display_name}" diff --git a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 index c3244fca4..8f00bdc1a 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 +++ b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 @@ -7,3 +7,6 @@ spec: sourceType: grpc image: {{ args.catalog_image }} displayName: {{ ctx.display_name }} +{% if args.publisher %} + publisher: {{ args.publisher }} +{% endif %} diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index d08376451..d604f8b02 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -36,6 +36,32 @@ def init(): run.init() config.init(Path(__file__).parent) +def list_vaults() -> list[str]: + """List all vaults (includes both mandatory and optional).""" + all_vaults = vault.phase_vault_list_all() + if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): + return [*all_vaults, "psap-rhoai-rc"] + + return all_vaults + + +def init_vaults_for_phase(phase: str) -> None: + mandatory_vaults = [ + *config.project.get_config("vaults.all", []), + *config.project.get_config(f"vaults.{phase}", []), + ] + optional_vaults = [ + *config.project.get_config("vaults.all-optional", []), + *config.project.get_config(f"vaults.{phase}-optional", []), + ] + + if phase == "prepare" and config.project.get_config( + "platform.rhoai.custom_catalog.enabled", False + ): + optional_vaults = [*optional_vaults, "psap-rhoai-rc"] + + vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) + @click.group(cls=ci_lib.HelpfulGroup) @click.pass_context @@ -49,7 +75,7 @@ def main(ctx): logger.info("No need to initialize the vaults for the resolve step") return - vault.phase_vault_init(ctx.invoked_subcommand) + init_vaults_for_phase(ctx.invoked_subcommand) @main.command() @@ -113,6 +139,7 @@ def post_cleanup(ctx) -> int: main.add_command( create_fournos_resolve_entrypoint( vault_list_funcs=[ + list_vaults, vault.phase_vault_list_all, caliper_export_list_vaults, caliper_export_list_optional_vaults, diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 44a941275..1d1f16985 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -50,10 +50,14 @@ operators: rhoai: custom_catalog: enabled: false - name: rhoai-custom + name: rhoai-catalog-dev namespace: openshift-marketplace image: null - display_name: Red Hat OpenShift AI Custom Catalog + display_name: Red Hat OpenShift AI + publisher: RHOAI Development Catalog + pull_secret: + vault: + name: psap-rhoai-rc namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml new file mode 100644 index 000000000..d18702599 --- /dev/null +++ b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml @@ -0,0 +1,9 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: quay-registry +spec: + repositoryDigestMirrors: + - source: registry.redhat.io/rhoai + mirrors: + - quay.io/rhoai diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 138c24509..7b9f31e36 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -1,8 +1,12 @@ from __future__ import annotations +import base64 import hashlib import json import logging +import tempfile +import time +from pathlib import Path from typing import Any from projects.cluster.toolbox.bootstrap_lws_operator import main as bootstrap_lws_operator @@ -35,6 +39,11 @@ ) logger = logging.getLogger(__name__) +RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" +RHOAI_PULL_SECRET_NAME = "pull-secret" +RHOAI_REGISTRY = "quay.io/rhoai" +RHOAI_PULL_SECRET_CONTENT = "rhoai_rc.secret" +RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: @@ -102,9 +111,130 @@ def deploy_rhoai_custom_catalog(*, rhoai: dict) -> int: catalog_namespace=custom_catalog["namespace"], catalog_image=custom_catalog["image"], display_name=custom_catalog.get("display_name", custom_catalog["name"]), + publisher=custom_catalog.get("publisher", ""), ) +def _custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: + pull_secret = custom_catalog.get("pull_secret", {}) + vault_spec = pull_secret.get("vault", {}) + vault_name = vault_spec.get("name") + content_name = vault_spec.get("content", RHOAI_PULL_SECRET_CONTENT) + + if not vault_name: + raise RuntimeError("RHOAI custom catalog pull_secret.vault.name is required") + + secret_path = vault.get_vault_content_path(vault_name, content_name) + if secret_path is None: + raise RuntimeError( + f"RHOAI pull secret content '{content_name}' was not found in vault '{vault_name}'" + ) + + return secret_path + + +def _decode_pull_secret(secret_data: dict[str, Any]) -> str: + encoded = secret_data.get("data", {}).get(".dockerconfigjson") + if not encoded: + raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") + + return base64.b64decode(encoded).decode("utf-8") + + +def wait_for_rhoai_pull_secret_ready( + *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 +) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + try: + secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded = _decode_pull_secret(secret) + if RHOAI_REGISTRY not in decoded: + raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + + mcp_status = oc( + "get", + "mcp", + "-o", + r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', + check=False, + log_stdout=False, + log_stderr=False, + ).stdout.strip() + if "False" in mcp_status: + raise RuntimeError("machine config pools are still updating") + + logger.info( + "RHOAI pull secret now contains %s and machine config pools are updated", + RHOAI_REGISTRY, + ) + return + except RuntimeError as exc: + last_error = str(exc) + logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) + time.sleep(poll_interval_seconds) + + raise RuntimeError( + f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + ) + + +def prepare_rhoai_pull_secret(custom_catalog: dict[str, Any]) -> None: + pull_secret_path = _custom_catalog_pull_secret_path(custom_catalog) + auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() + if not auth_basic: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + + if RHOAI_REGISTRY in decoded_secret: + logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + return + + logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: + temp_file.write(decoded_secret) + temp_path = Path(temp_file.name) + + try: + oc( + "registry", + "login", + f"--registry={RHOAI_REGISTRY}", + f"--auth-basic={auth_basic}", + "--to", + str(temp_path), + ) + oc( + "set", + "data", + f"secret/{RHOAI_PULL_SECRET_NAME}", + "-n", + RHOAI_PULL_SECRET_NAMESPACE, + f"--from-file=.dockerconfigjson={temp_path}", + ) + finally: + temp_path.unlink(missing_ok=True) + + wait_for_rhoai_pull_secret_ready() + + +def apply_rhoai_icsp() -> None: + logger.info("Applying RHOAI ICSP manifest: %s", RHOAI_ICSP_MANIFEST) + oc("apply", "-f", str(RHOAI_ICSP_MANIFEST)) + + def rhoai_operator_spec( *, rhoai: dict, @@ -179,7 +309,11 @@ def prepare_gpu_operator() -> None: def prepare_rhoai_operator() -> None: platform = runtime_config.get_platform_config() + custom_catalog = platform["rhoai"]["custom_catalog"] prepare_rhcl_operator() + if custom_catalog["enabled"]: + prepare_rhoai_pull_secret(custom_catalog) + apply_rhoai_icsp() deploy_rhoai_custom_catalog(rhoai=platform["rhoai"]) operator_spec = operator_spec_by_package(platform, "rhods-operator") operator_spec = rhoai_operator_spec(rhoai=platform["rhoai"], operator_spec=operator_spec) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index a798fb8e2..e2cc81df6 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -8,7 +8,7 @@ from projects.core.library import config as core_config from projects.core.library import env from projects.llm_d.orchestration import ci as llmd_ci -from projects.llm_d.orchestration import runtime_config, test_phase +from projects.llm_d.orchestration import prepare_phase, runtime_config, test_phase from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) @@ -198,6 +198,114 @@ def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided assert runtime_config.get_benchmark_keys() == ["short"] +def test_list_vaults_only_includes_rhoai_rc_vault_for_custom_catalog_runs() -> None: + _init_project_config() + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + assert "psap-rhoai-rc" not in llmd_ci.list_vaults() + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + assert "psap-rhoai-rc" in llmd_ci.list_vaults() + + +def test_prepare_phase_adds_rhoai_rc_vault_only_for_custom_catalog_runs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + calls: list[dict[str, object]] = [] + + def _fake_init(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(llmd_ci.vault, "init", _fake_init) + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + llmd_ci.init_vaults_for_phase("prepare") + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + llmd_ci.init_vaults_for_phase("prepare") + + assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] + assert "psap-rhoai-rc" in calls[1]["optional_vaults"] + + +def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + platform = { + "operators": { + "rhcl-operator": { + "package": "rhcl-operator", + "namespace": "openshift-operators", + "source": "redhat-operators", + "channel": "stable", + }, + "rhods-operator": { + "package": "rhods-operator", + "namespace": "redhat-ods-operator", + "source": "redhat-operators", + "channel": "stable-3.x", + }, + }, + "rhoai": { + "custom_catalog": { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": {"vault": {"name": "psap-rhoai-rc"}}, + }, + "namespace": "redhat-ods-applications", + "datasciencecluster_name": "default-dsc", + "components": ["kserve"], + "required_crds_before_dsc": ["datascienceclusters.datasciencecluster.opendatahub.io"], + "required_crds_after_dsc": ["llminferenceservices.serving.kserve.io"], + }, + } + + calls: list[object] = [] + + monkeypatch.setattr(prepare_phase.runtime_config, "get_platform_config", lambda: platform) + monkeypatch.setattr(prepare_phase, "prepare_rhcl_operator", lambda: calls.append("rhcl")) + monkeypatch.setattr( + prepare_phase, + "prepare_rhoai_pull_secret", + lambda custom_catalog: calls.append(("pull", custom_catalog["image"])), + ) + monkeypatch.setattr(prepare_phase, "apply_rhoai_icsp", lambda: calls.append("icsp")) + monkeypatch.setattr( + prepare_phase, + "deploy_rhoai_custom_catalog", + lambda rhoai: calls.append(("catalog", rhoai["custom_catalog"]["publisher"])), + ) + monkeypatch.setattr( + prepare_phase, + "ensure_operator_subscription", + lambda operator_spec: calls.append(("sub", operator_spec["package"])), + ) + monkeypatch.setattr( + prepare_phase, + "ensure_required_crds_before_dsc", + lambda: calls.append("crds"), + ) + + prepare_phase.prepare_rhoai_operator() + + assert calls == [ + "rhcl", + ("pull", "quay.io/rhoai/rhoai-fbc-fragment@sha256:test"), + "icsp", + ("catalog", "RHOAI Development Catalog"), + ("sub", "rhods-operator"), + "crds", + ] + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/vaults/psap-rhoai-rc.yaml b/vaults/psap-rhoai-rc.yaml new file mode 100644 index 000000000..f9bbc10c4 --- /dev/null +++ b/vaults/psap-rhoai-rc.yaml @@ -0,0 +1,8 @@ +env_key: PSAP_RHOAI_RC_SECRET_PATH +description: | + Vault containing credentials for the RHOAI release-candidate registry + +content: + rhoai_rc.secret: + description: | + Basic auth string used to extend the cluster pull secret for quay.io/rhoai From 27f0b0686074bd4307479243fc28ad2bf2712001 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 9 Jul 2026 10:55:31 +0100 Subject: [PATCH 0002/1072] fix: Address review comments Signed-off-by: Alberto Perdomo --- .../orchestration/config.d/platform.yaml | 1 + projects/llm_d/orchestration/prepare_phase.py | 218 +------------- projects/llm_d/tests/test_profiles.py | 53 +++- projects/rhoai/library/deploy.py | 276 ++++++++++++++++++ vaults/psap-rhoai-rc.yaml | 2 + 5 files changed, 330 insertions(+), 220 deletions(-) create mode 100644 projects/rhoai/library/deploy.py diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 1d1f16985..f9b2975e2 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -58,6 +58,7 @@ rhoai: pull_secret: vault: name: psap-rhoai-rc + content: rhoai_rc.secret namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 7b9f31e36..3e3b63047 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -1,17 +1,11 @@ from __future__ import annotations -import base64 import hashlib import json import logging -import tempfile -import time from pathlib import Path -from typing import Any from projects.cluster.toolbox.bootstrap_lws_operator import main as bootstrap_lws_operator -from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator -from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.dsl.utils.k8s import ( @@ -33,32 +27,22 @@ run as capture_prepare_state_toolbox_run, ) from projects.llm_d.toolbox.ensure_gateway import main as ensure_gateway_command +from projects.rhoai.library.deploy import ( + ensure_operator_subscription, + operator_spec_by_package, +) +from projects.rhoai.library.deploy import ( + prepare_rhoai_operator as rhoai_prepare_rhoai_operator, +) from projects.rhoai.toolbox.apply_datasciencecluster import main as apply_datasciencecluster_command from projects.rhoai.toolbox.wait_datasciencecluster_ready import ( main as wait_datasciencecluster_ready_command, ) logger = logging.getLogger(__name__) -RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" -RHOAI_PULL_SECRET_NAME = "pull-secret" -RHOAI_REGISTRY = "quay.io/rhoai" -RHOAI_PULL_SECRET_CONTENT = "rhoai_rc.secret" RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" -def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: - operators = platform["operators"] - if isinstance(operators, dict): - if package in operators: - return {"package": package, **operators[package]} - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") - - for operator_spec in operators: - if operator_spec["package"] == package: - return operator_spec - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") - - def verify_oc_access() -> None: oc("whoami") @@ -85,177 +69,11 @@ def verify_cluster_version() -> None: ) -def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: - return cluster_deploy_operator.run( - package_name=operator_spec["package"], - target_namespace=operator_spec["namespace"], - source_name=operator_spec["source"], - channel=operator_spec["channel"], - source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), - display_name=operator_spec.get("display_name", operator_spec["package"]), - artifact_dirname_suffix=f"_{operator_spec['package']}", - ) - - -def deploy_rhoai_custom_catalog(*, rhoai: dict) -> int: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - logger.info("RHOAI custom catalog disabled; using default catalog source") - return 0 - - if not custom_catalog.get("image"): - raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") - - return deploy_custom_catalog.run( - catalog_source_name=custom_catalog["name"], - catalog_namespace=custom_catalog["namespace"], - catalog_image=custom_catalog["image"], - display_name=custom_catalog.get("display_name", custom_catalog["name"]), - publisher=custom_catalog.get("publisher", ""), - ) - - -def _custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: - pull_secret = custom_catalog.get("pull_secret", {}) - vault_spec = pull_secret.get("vault", {}) - vault_name = vault_spec.get("name") - content_name = vault_spec.get("content", RHOAI_PULL_SECRET_CONTENT) - - if not vault_name: - raise RuntimeError("RHOAI custom catalog pull_secret.vault.name is required") - - secret_path = vault.get_vault_content_path(vault_name, content_name) - if secret_path is None: - raise RuntimeError( - f"RHOAI pull secret content '{content_name}' was not found in vault '{vault_name}'" - ) - - return secret_path - - -def _decode_pull_secret(secret_data: dict[str, Any]) -> str: - encoded = secret_data.get("data", {}).get(".dockerconfigjson") - if not encoded: - raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") - - return base64.b64decode(encoded).decode("utf-8") - - -def wait_for_rhoai_pull_secret_ready( - *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 -) -> None: - deadline = time.monotonic() + timeout_seconds - last_error: str | None = None - - while time.monotonic() < deadline: - try: - secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded = _decode_pull_secret(secret) - if RHOAI_REGISTRY not in decoded: - raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") - - mcp_status = oc( - "get", - "mcp", - "-o", - r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', - check=False, - log_stdout=False, - log_stderr=False, - ).stdout.strip() - if "False" in mcp_status: - raise RuntimeError("machine config pools are still updating") - - logger.info( - "RHOAI pull secret now contains %s and machine config pools are updated", - RHOAI_REGISTRY, - ) - return - except RuntimeError as exc: - last_error = str(exc) - logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) - time.sleep(poll_interval_seconds) - - raise RuntimeError( - f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" - ) - - -def prepare_rhoai_pull_secret(custom_catalog: dict[str, Any]) -> None: - pull_secret_path = _custom_catalog_pull_secret_path(custom_catalog) - auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() - if not auth_basic: - raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") - - current_secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded_secret = _decode_pull_secret(current_secret) - - if RHOAI_REGISTRY in decoded_secret: - logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) - return - - logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: - temp_file.write(decoded_secret) - temp_path = Path(temp_file.name) - - try: - oc( - "registry", - "login", - f"--registry={RHOAI_REGISTRY}", - f"--auth-basic={auth_basic}", - "--to", - str(temp_path), - ) - oc( - "set", - "data", - f"secret/{RHOAI_PULL_SECRET_NAME}", - "-n", - RHOAI_PULL_SECRET_NAMESPACE, - f"--from-file=.dockerconfigjson={temp_path}", - ) - finally: - temp_path.unlink(missing_ok=True) - - wait_for_rhoai_pull_secret_ready() - - def apply_rhoai_icsp() -> None: logger.info("Applying RHOAI ICSP manifest: %s", RHOAI_ICSP_MANIFEST) oc("apply", "-f", str(RHOAI_ICSP_MANIFEST)) -def rhoai_operator_spec( - *, - rhoai: dict, - operator_spec: dict[str, str], -) -> dict[str, str]: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - return operator_spec - - updated_spec = dict(operator_spec) - updated_spec["source"] = custom_catalog["name"] - updated_spec["source_namespace"] = custom_catalog["namespace"] - return updated_spec - - -def prepare_rhcl_operator() -> None: - platform = runtime_config.get_platform_config() - operator_spec = operator_spec_by_package(platform, "rhcl-operator") - ensure_operator_subscription(operator_spec) - - def prepare_cert_manager() -> None: platform = runtime_config.get_platform_config() operator_spec = operator_spec_by_package(platform, "openshift-cert-manager-operator") @@ -309,24 +127,10 @@ def prepare_gpu_operator() -> None: def prepare_rhoai_operator() -> None: platform = runtime_config.get_platform_config() - custom_catalog = platform["rhoai"]["custom_catalog"] - prepare_rhcl_operator() - if custom_catalog["enabled"]: - prepare_rhoai_pull_secret(custom_catalog) - apply_rhoai_icsp() - deploy_rhoai_custom_catalog(rhoai=platform["rhoai"]) - operator_spec = operator_spec_by_package(platform, "rhods-operator") - operator_spec = rhoai_operator_spec(rhoai=platform["rhoai"], operator_spec=operator_spec) - ensure_operator_subscription(operator_spec) - ensure_required_crds_before_dsc() - - -def ensure_required_crds_before_dsc() -> None: - platform = runtime_config.get_platform_config() - rhoai = platform["rhoai"] - wait_for_crds_command.run( - crd_names=rhoai["required_crds_before_dsc"], - display_name="RHOAI pre-DSC CRDs", + rhoai_prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=apply_rhoai_icsp, ) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index e2cc81df6..5b4148816 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -8,10 +8,11 @@ from projects.core.library import config as core_config from projects.core.library import env from projects.llm_d.orchestration import ci as llmd_ci -from projects.llm_d.orchestration import prepare_phase, runtime_config, test_phase +from projects.llm_d.orchestration import runtime_config, test_phase from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) +from projects.rhoai.library import deploy as rhoai_deploy PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" @@ -258,7 +259,7 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", "display_name": "Red Hat OpenShift AI", "publisher": "RHOAI Development Catalog", - "pull_secret": {"vault": {"name": "psap-rhoai-rc"}}, + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, }, "namespace": "redhat-ods-applications", "datasciencecluster_name": "default-dsc", @@ -270,31 +271,35 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy calls: list[object] = [] - monkeypatch.setattr(prepare_phase.runtime_config, "get_platform_config", lambda: platform) - monkeypatch.setattr(prepare_phase, "prepare_rhcl_operator", lambda: calls.append("rhcl")) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "prepare_rhcl_operator", lambda platform: calls.append("rhcl") + ) + monkeypatch.setattr( + rhoai_deploy, "prepare_rhoai_pull_secret", - lambda custom_catalog: calls.append(("pull", custom_catalog["image"])), + lambda custom_catalog: calls.append(("pull", custom_catalog.image)), ) - monkeypatch.setattr(prepare_phase, "apply_rhoai_icsp", lambda: calls.append("icsp")) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "deploy_rhoai_custom_catalog", - lambda rhoai: calls.append(("catalog", rhoai["custom_catalog"]["publisher"])), + lambda custom_catalog: calls.append(("catalog", custom_catalog.publisher)), ) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "ensure_operator_subscription", lambda operator_spec: calls.append(("sub", operator_spec["package"])), ) monkeypatch.setattr( - prepare_phase, + rhoai_deploy, "ensure_required_crds_before_dsc", - lambda: calls.append("crds"), + lambda rhoai: calls.append("crds"), ) - prepare_phase.prepare_rhoai_operator() + rhoai_deploy.prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=lambda: calls.append("icsp"), + ) assert calls == [ "rhcl", @@ -306,6 +311,28 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy ] +def test_custom_catalog_pull_secret_path_uses_explicit_vault_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, str] = {} + + def _fake_get_vault_content_path(vault_name: str, content_name: str): + captured["vault_name"] = vault_name + captured["content_name"] = content_name + return Path("/tmp/rhoai_rc.secret") + + monkeypatch.setattr(rhoai_deploy.vault, "get_vault_content_path", _fake_get_vault_content_path) + + path = rhoai_deploy.custom_catalog_pull_secret_path( + { + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + } + ) + + assert path == Path("/tmp/rhoai_rc.secret") + assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py new file mode 100644 index 000000000..220ec8854 --- /dev/null +++ b/projects/rhoai/library/deploy.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import base64 +import logging +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator +from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog +from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command +from projects.core.dsl.utils.k8s import oc, oc_get_json +from projects.core.library import vault + +logger = logging.getLogger(__name__) + +RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" +RHOAI_PULL_SECRET_NAME = "pull-secret" +RHOAI_REGISTRY = "quay.io/rhoai" + + +class _RhoaiCustomCatalogPullSecretVaultConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + content: str + + +class _RhoaiCustomCatalogPullSecretConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + vault: _RhoaiCustomCatalogPullSecretVaultConfig + + +class _RhoaiCustomCatalogPullSecretInput(BaseModel): + model_config = ConfigDict(extra="ignore") + + pull_secret: _RhoaiCustomCatalogPullSecretConfig + + +class RhoaiCustomCatalogConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + enabled: bool = False + name: str + namespace: str + image: str | None = None + display_name: str | None = None + publisher: str | None = None + pull_secret: _RhoaiCustomCatalogPullSecretConfig + + +class RhoaiOperatorConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + custom_catalog: RhoaiCustomCatalogConfig + namespace: str + datasciencecluster_name: str + components: list[str] = Field(default_factory=list) + required_crds_before_dsc: list[str] = Field(default_factory=list) + required_crds_after_dsc: list[str] = Field(default_factory=list) + + +def custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: + catalog = _RhoaiCustomCatalogPullSecretInput.model_validate(custom_catalog) + secret_path = vault.get_vault_content_path( + catalog.pull_secret.vault.name, + catalog.pull_secret.vault.content, + ) + if secret_path is None: + raise RuntimeError( + "RHOAI pull secret content " + f"'{catalog.pull_secret.vault.content}' was not found in vault " + f"'{catalog.pull_secret.vault.name}'" + ) + return secret_path + + +def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: + operators = platform["operators"] + if isinstance(operators, dict): + if package in operators: + return {"package": package, **operators[package]} + raise KeyError(f"Unknown operator package: {package}") + + for operator_spec in operators: + if operator_spec["package"] == package: + return operator_spec + raise KeyError(f"Unknown operator package: {package}") + + +def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: + return cluster_deploy_operator.run( + package_name=operator_spec["package"], + target_namespace=operator_spec["namespace"], + source_name=operator_spec["source"], + channel=operator_spec["channel"], + source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), + display_name=operator_spec.get("display_name", operator_spec["package"]), + artifact_dirname_suffix=f"_{operator_spec['package']}", + ) + + +def deploy_rhoai_custom_catalog(*, custom_catalog: RhoaiCustomCatalogConfig) -> int: + if not custom_catalog.enabled: + logger.info("RHOAI custom catalog disabled; using default catalog source") + return 0 + + if not custom_catalog.image: + raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") + + return deploy_custom_catalog.run( + catalog_source_name=custom_catalog.name, + catalog_namespace=custom_catalog.namespace, + catalog_image=custom_catalog.image, + display_name=custom_catalog.display_name or custom_catalog.name, + publisher=custom_catalog.publisher or "", + ) + + +def rhoai_operator_spec( + *, + custom_catalog: RhoaiCustomCatalogConfig, + operator_spec: dict[str, str], +) -> dict[str, str]: + if not custom_catalog.enabled: + return operator_spec + + updated_spec = dict(operator_spec) + updated_spec["source"] = custom_catalog.name + updated_spec["source_namespace"] = custom_catalog.namespace + return updated_spec + + +def prepare_rhcl_operator(platform: dict[str, Any]) -> None: + operator_spec = operator_spec_by_package(platform, "rhcl-operator") + ensure_operator_subscription(operator_spec) + + +def _decode_pull_secret(secret_data: dict[str, Any]) -> str: + encoded = secret_data.get("data", {}).get(".dockerconfigjson") + if not encoded: + raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") + + return base64.b64decode(encoded).decode("utf-8") + + +def wait_for_rhoai_pull_secret_ready( + *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 +) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + try: + secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded = _decode_pull_secret(secret) + if RHOAI_REGISTRY not in decoded: + raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + + mcp_status = oc( + "get", + "mcp", + "-o", + r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', + check=False, + log_stdout=False, + log_stderr=False, + ).stdout.strip() + if "False" in mcp_status: + raise RuntimeError("machine config pools are still updating") + + logger.info( + "RHOAI pull secret now contains %s and machine config pools are updated", + RHOAI_REGISTRY, + ) + return + except RuntimeError as exc: + last_error = str(exc) + logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) + time.sleep(poll_interval_seconds) + + raise RuntimeError( + f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + ) + + +def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: + pull_secret_path = vault.get_vault_content_path( + custom_catalog.pull_secret.vault.name, + custom_catalog.pull_secret.vault.content, + ) + if pull_secret_path is None: + raise RuntimeError( + "RHOAI pull secret content " + f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " + f"'{custom_catalog.pull_secret.vault.name}'" + ) + auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() + if not auth_basic: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + + if RHOAI_REGISTRY in decoded_secret: + logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + return + + logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: + temp_file.write(decoded_secret) + temp_path = Path(temp_file.name) + + try: + oc( + "registry", + "login", + f"--registry={RHOAI_REGISTRY}", + f"--auth-basic={auth_basic}", + "--to", + str(temp_path), + ) + oc( + "set", + "data", + f"secret/{RHOAI_PULL_SECRET_NAME}", + "-n", + RHOAI_PULL_SECRET_NAMESPACE, + f"--from-file=.dockerconfigjson={temp_path}", + ) + finally: + temp_path.unlink(missing_ok=True) + + wait_for_rhoai_pull_secret_ready() + + +def ensure_required_crds_before_dsc(rhoai: RhoaiOperatorConfig) -> None: + wait_for_crds_command.run( + crd_names=rhoai.required_crds_before_dsc, + display_name="RHOAI pre-DSC CRDs", + ) + + +def prepare_rhoai_operator( + *, + platform: dict[str, Any], + rhoai: dict[str, Any], + icsp_applier: Callable[[], None], +) -> None: + rhoai_config = RhoaiOperatorConfig.model_validate(rhoai) + + prepare_rhcl_operator(platform) + if rhoai_config.custom_catalog.enabled: + prepare_rhoai_pull_secret(rhoai_config.custom_catalog) + icsp_applier() + deploy_rhoai_custom_catalog(custom_catalog=rhoai_config.custom_catalog) + operator_spec = operator_spec_by_package(platform, "rhods-operator") + operator_spec = rhoai_operator_spec( + custom_catalog=rhoai_config.custom_catalog, + operator_spec=operator_spec, + ) + ensure_operator_subscription(operator_spec) + ensure_required_crds_before_dsc(rhoai_config) diff --git a/vaults/psap-rhoai-rc.yaml b/vaults/psap-rhoai-rc.yaml index f9bbc10c4..e35cd4560 100644 --- a/vaults/psap-rhoai-rc.yaml +++ b/vaults/psap-rhoai-rc.yaml @@ -6,3 +6,5 @@ content: rhoai_rc.secret: description: | Basic auth string used to extend the cluster pull secret for quay.io/rhoai + sample: | + rhoai+rhoai_external_readonly_bot:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx From 87b0035a1cd3809ad1ec65e5f69d2736f9f5bf8f Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 9 Jul 2026 11:12:47 +0100 Subject: [PATCH 0003/1072] fix: MCP status check passing on empty output Signed-off-by: Alberto Perdomo --- projects/llm_d/tests/test_profiles.py | 31 +++++++++++++++++++++++++++ projects/rhoai/library/deploy.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 5b4148816..97a03cdfb 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -333,6 +333,37 @@ def _fake_get_vault_content_path(vault_name: str, content_name: str): assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} +def test_wait_for_rhoai_pull_secret_ready_treats_empty_mcp_status_as_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + mcp_outputs = iter(["", "True"]) + clock = {"value": 0.0} + + def _fake_monotonic() -> float: + value = clock["value"] + clock["value"] += 0.1 + return value + + def _fake_sleep(_seconds: float) -> None: + clock["value"] += 0.1 + + def _fake_oc_get_json(*args, **kwargs): + return {"data": {".dockerconfigjson": "unused"}} + + def _fake_oc(*args, **kwargs): + if args[:2] == ("get", "mcp"): + return type("Result", (), {"stdout": next(mcp_outputs), "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy, "_decode_pull_secret", lambda _secret: "quay.io/rhoai") + monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) + monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) + + rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 220ec8854..fa4c6203c 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -175,7 +175,7 @@ def wait_for_rhoai_pull_secret_ready( log_stdout=False, log_stderr=False, ).stdout.strip() - if "False" in mcp_status: + if not mcp_status or "False" in mcp_status: raise RuntimeError("machine config pools are still updating") logger.info( From c4854cfc2e3a5d8bd390d3145ec953c43f986907 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 9 Jul 2026 15:43:27 -0400 Subject: [PATCH 0004/1072] fix: update caliper postprocess config fields after PR 110 schema change PR 110 renamed caliper.postprocess.kpi fields: - kpi.generate -> kpi.artifacts_to_kpis - kpi.export -> removed (no longer needed) - analyze.baseline -> analyze.historical_kpis These three project configs were not updated, causing pydantic ValidationError (extra_forbidden) during postprocessing. Co-authored-by: Cursor --- projects/llm_d_legacy/testing/config.yaml | 6 ++---- projects/mcp_gateway/orchestration/config.yaml | 6 ++---- projects/rhaiis/orchestration/config.yaml | 4 +--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/projects/llm_d_legacy/testing/config.yaml b/projects/llm_d_legacy/testing/config.yaml index b16b0008f..dd4050ccf 100644 --- a/projects/llm_d_legacy/testing/config.yaml +++ b/projects/llm_d_legacy/testing/config.yaml @@ -364,13 +364,11 @@ caliper: kpi: enabled: false - generate: + artifacts_to_kpis: enabled: false output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 34f67a0a9..b73f7da9d 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -34,14 +34,12 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json export: diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 045227198..6c932e4bb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -31,11 +31,9 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false export: from: null From 18eab4e4e6e54dd4a974b6a0a4db3016af253423 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 9 Jul 2026 16:33:14 -0400 Subject: [PATCH 0005/1072] docs: document available pipelines and spec.env variables Add pipeline reference table and spec.env documentation (PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_SHA) to the gitops README. Co-authored-by: Cursor --- fournos/gitops/README.md | 52 +++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/fournos/gitops/README.md b/fournos/gitops/README.md index 5f1821b3e..4eae41404 100644 --- a/fournos/gitops/README.md +++ b/fournos/gitops/README.md @@ -37,12 +37,52 @@ gitops/ - **Build**: Shipwright build configuration for building forge image from source - **BuildRun**: Triggers the build process -### Workflows -- **Task**: `forge-step` - Reusable Tekton task for executing forge commands -- **Pipelines**: Multiple pipeline variants: - - `forge-full`: Complete pipeline with pre-cleanup, prepare, test, export-artifacts, and post-cleanup - - `pipeline-test-only`: Test execution only - - `pipeline-replot`: Replotting functionality +### Workflows +- **Task**: `forge-step` — reusable Tekton task for executing forge commands +- **Pipelines**: + +| Pipeline | Steps | Finally | Use case | +|---|---|---|---| +| `forge-full` | pre-cleanup → prepare → preflight → test | post-cleanup → export-artifacts | Full lifecycle with cleanup on both ends | +| `forge-prepare-test` | prepare → preflight → test | export-artifacts | Prepare + test without cleanup | +| `forge-test-only` | preflight → test | export-artifacts | Skip preparation, just run the test | +| `forge-prepare-only` | prepare → preflight | export-artifacts | Set up the cluster without benchmarks | +| `forge-replot` | replot | export-artifacts | Re-run visualization/postprocessing | + +Select a pipeline via `spec.pipeline` in the FournosJob CR (default: `fournos-full`). + +### FournosJob `spec.env` variables + +The `forge-step` task bootstrap script reads certain well-known variables +from `spec.env` and acts on them before launching the execution engine. + +#### Git checkout + +| Variable | Description | +|---|---| +| `PULL_PULL_SHA` | Checkout a specific git commit SHA. The task runs `git fetch origin ` and `git reset --hard FETCH_HEAD` before executing. | +| `PULL_NUMBER` | Checkout the HEAD of a GitHub PR (`refs/pull//head`). Also enables GitHub PR notifications — test results are posted as a comment on the PR. | +| `PULL_BASE_SHA` | Base branch commit SHA (informational context for the execution engine). | + +If neither `PULL_PULL_SHA` nor `PULL_NUMBER` is set, the execution engine +runs with the code baked into the container image. + +`PULL_NUMBER` is the most convenient option for testing a PR: it checks +out the latest code from that PR **and** posts results back to GitHub. + +#### Example + +```yaml +spec: + env: + PULL_NUMBER: "118" # checkout PR #118 HEAD + post results to PR +``` + +```yaml +spec: + env: + PULL_PULL_SHA: "abc123" # checkout exact commit, no PR notification +``` ## Environments From 2a9f27f0e93972e8ecca7ecf4745c76124a721e9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:17 +0200 Subject: [PATCH 0006/1072] [caliper] s3: remove the import/export prefix config field --- .../caliper/cli/orchestration_entrypoints.py | 8 +++---- projects/caliper/cli/s3_export.py | 21 ++++--------------- .../orchestration/postprocess_config.py | 8 ------- .../caliper/orchestration/step_logging.py | 13 +++++++++++- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 41203348e..5e8fba7d4 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -75,12 +75,10 @@ def s3_import_entrypoint( s3_parent_config = postprocess_config.s3 s3_config = postprocess_config.s3.import_ - # Build import prefix using instance + s3.prefix + directory + # Build import prefix using instance + directory import_prefix_parts = [] if s3_parent_config.instance: import_prefix_parts.append(s3_parent_config.instance) - if s3_parent_config.prefix: - import_prefix_parts.append(s3_parent_config.prefix.rstrip("/")) if s3_parent_config.directory: import_prefix_parts.append(s3_parent_config.directory) @@ -89,7 +87,8 @@ def s3_import_entrypoint( log_s3_import_command( bucket=s3_parent_config.bucket, - prefix=import_prefix, + instance=s3_parent_config.instance, + directory=s3_parent_config.directory, output_dir=import_dir, include_kpis_json=s3_config.include_kpis_json, include_kpis_csv=s3_config.include_kpis_csv, @@ -139,7 +138,6 @@ def s3_export_entrypoint( ai_data_dir=ai_data_dir, analysis_file=analysis_file, bucket=s3_parent_config.bucket, - prefix=s3_parent_config.prefix, instance=s3_parent_config.instance, directory=s3_parent_config.directory, upload_id=s3_config.upload_id, diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index c60bd9d36..299305f17 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -224,7 +224,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path | None = None, analysis_file: Path | None = None, bucket: str, - prefix: str = "", instance: str | None = None, directory: str | None = None, upload_id: str | None = None, @@ -240,7 +239,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path to AI data directory to upload analysis_file: Path to analysis file to upload bucket: S3 bucket name - prefix: S3 object prefix/path instance: Instance identifier for S3 organization directory: Directory identifier for S3 organization upload_id: Custom upload identifier (uses timestamp if not provided) @@ -288,12 +286,10 @@ def run_s3_export_with_explicit_paths( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ s3_path_components = [] if instance: s3_path_components.append(instance) - if prefix: - s3_path_components.append(prefix.rstrip("/")) if directory: s3_path_components.append(directory) s3_path_components.append(upload_id) @@ -302,9 +298,7 @@ def run_s3_export_with_explicit_paths( logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - logger.info( - f"Path structure: {instance}/{prefix.rstrip('/') if prefix else 'no-prefix'}/{directory}/{upload_id}/" - ) + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") @@ -601,12 +595,10 @@ def run_s3_export( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ s3_path_components = [] if instance: s3_path_components.append(instance) - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix: - s3_path_components.append(s3_parent_config.prefix.rstrip("/")) if directory: s3_path_components.append(directory) s3_path_components.append(upload_id) @@ -615,12 +607,7 @@ def run_s3_export( logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - prefix_component = ( - s3_parent_config.prefix.rstrip("/") - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix - else "no-prefix" - ) - logger.info(f"Path structure: {instance}/{prefix_component}/{directory}/{upload_id}/") + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index 4d1ca6815..b30ccb222 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -167,10 +167,6 @@ class CaliperOrchestrationS3ExportSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False - prefix: str | None = Field( - default=None, - description="S3 key prefix/folder path (optional).", - ) upload_id: str | None = Field( default=None, description="Custom upload identifier; uses timestamp (YY-MM-DD_HHMMSS) if null.", @@ -225,10 +221,6 @@ class CaliperOrchestrationS3Section(BaseModel): default=None, description="Directory identifier for S3 organization (required when import or export is enabled).", ) - prefix: str | None = Field( - default=None, - description="S3 prefix for organizing uploads and imports.", - ) vault: CaliperOrchestrationS3VaultSection = Field( default_factory=CaliperOrchestrationS3VaultSection, description="Vault configuration for AWS credentials.", diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 3aeae431b..474ed72c4 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -390,7 +390,8 @@ def log_ai_data_command( def log_s3_import_command( bucket: str, - prefix: str, + instance: str | None, + directory: str | None, output_dir: Path, include_kpis_json: bool = False, include_kpis_csv: bool = False, @@ -398,6 +399,14 @@ def log_s3_import_command( max_downloads: int = 50, ) -> None: """Log the CLI command to reproduce the S3 import step.""" + # Build prefix from instance and directory + prefix_parts = [] + if instance: + prefix_parts.append(instance) + if directory: + prefix_parts.append(directory) + prefix = "/".join(prefix_parts) if prefix_parts else "" + command_parts = [ f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' ] @@ -416,6 +425,8 @@ def log_s3_import_command( step_args = { "bucket": bucket, + "instance": instance, + "directory": directory, "prefix": prefix, "output_dir": str(output_dir), "include_kpis_json": include_kpis_json, From d3031b414a0c758a525e034aadc73380c88e0e1b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:30 +0200 Subject: [PATCH 0007/1072] [llm_d] orchestration: config: remove the s3 config field --- projects/llm_d/orchestration/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 56265eb55..36f12e78f 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -78,7 +78,6 @@ caliper: vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials - prefix: llm_d/ import: enabled: true output_dir: historical_data From f8f29963adb93b8595f577420d143b1f20a4e492 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:41:48 +0200 Subject: [PATCH 0008/1072] [skeleton] orchestration: config: remove the prefix config field --- projects/skeleton/orchestration/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 4c15f6591..91f2a4761 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -86,7 +86,6 @@ caliper: bucket: psap-dashboard-data instance: smoke directory: skeleton-dashboard - prefix: skeleton/ vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials From beb882acb9c9b34253f55dc14548d17ba76a7ed4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:57:01 +0200 Subject: [PATCH 0009/1072] .github: workflows: update workflows to actions/checkout@v5 --- .github/steps/prepare_minimal_ubuntu_image/action.yml | 1 + .github/workflows/check_consistency.yml | 3 ++- .github/workflows/check_python_syntax.yml | 3 ++- .github/workflows/image-push.yaml | 4 +++- .github/workflows/ruff.yaml | 4 +++- .github/workflows/test_toolbox_dsl.yml | 4 +++- 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/steps/prepare_minimal_ubuntu_image/action.yml b/.github/steps/prepare_minimal_ubuntu_image/action.yml index ebbedd53c..4570b25fc 100644 --- a/.github/steps/prepare_minimal_ubuntu_image/action.yml +++ b/.github/steps/prepare_minimal_ubuntu_image/action.yml @@ -1,4 +1,5 @@ name: "Prepare FORGE minimal ubuntu image" +description: "Install minimal Python dependencies required for FORGE repository operations" runs: using: "composite" # <-- this is the important part diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index 04556928e..0fd25b2a5 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -18,9 +18,10 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/steps/prepare_minimal_ubuntu_image diff --git a/.github/workflows/check_python_syntax.yml b/.github/workflows/check_python_syntax.yml index d13fd5ca4..41e9f3c40 100644 --- a/.github/workflows/check_python_syntax.yml +++ b/.github/workflows/check_python_syntax.yml @@ -10,9 +10,10 @@ jobs: max-parallel: 4 steps: - name: Use checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Check that all the files can be compiled run: | diff --git a/.github/workflows/image-push.yaml b/.github/workflows/image-push.yaml index 4cd5141f2..d4a880269 100644 --- a/.github/workflows/image-push.yaml +++ b/.github/workflows/image-push.yaml @@ -10,7 +10,9 @@ jobs: image-push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - name: Remove GitHub credentials from git config run: | diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index 3f4200572..6fea10352 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -16,7 +16,9 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/test_toolbox_dsl.yml b/.github/workflows/test_toolbox_dsl.yml index 669fdafc6..66ac3c418 100644 --- a/.github/workflows/test_toolbox_dsl.yml +++ b/.github/workflows/test_toolbox_dsl.yml @@ -20,7 +20,9 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: From d21146f2789350e2e557b1a44b48cd65d4df8241 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 13:57:37 +0200 Subject: [PATCH 0010/1072] .github: workflows: check_consistency: enforce EOL at EOF --- .github/workflows/check_consistency.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index 0fd25b2a5..e447d86e9 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -27,3 +27,22 @@ jobs: - name: Check that no symlink is broken run: bin/run_ansible_toolbox.py repo validate_no_broken_link + + - name: Check that all source files end with EOL + run: | + # Find all source files and check they end with newline + files_without_eol=() + + while IFS= read -r -d '' file; do + if [[ -s "$file" ]] && [[ -n "$(tail -c1 "$file")" ]]; then + files_without_eol+=("$file") + fi + done < <(find . -path './.git' -prune -o \( -name "*.py" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" -o -name "*.md" \) -type f -print0) + + if [[ ${#files_without_eol[@]} -gt 0 ]]; then + echo "❌ Files missing final newline:" + printf '%s\n' "${files_without_eol[@]}" + echo "Fix with: echo >> filename" + exit 1 + fi + echo "✅ All source files end with newline" From 312652972e12ec2272cdab0713759dcad3c39ecd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 14:02:01 +0200 Subject: [PATCH 0011/1072] [projects] Add EOL at EOF --- projects/cluster/toolbox/rebuild_image/README.md | 2 +- projects/core/tests/README.md | 2 +- .../llm_d/orchestration/manifests/leaderworkersetoperator.yaml | 2 +- projects/matrix_benchmarking/subproject/PLUGINS.md | 2 +- projects/mcp_gateway/orchestration/presets.d/presets.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/cluster/toolbox/rebuild_image/README.md b/projects/cluster/toolbox/rebuild_image/README.md index b418ba72a..324f2ea26 100644 --- a/projects/cluster/toolbox/rebuild_image/README.md +++ b/projects/cluster/toolbox/rebuild_image/README.md @@ -55,4 +55,4 @@ result = rebuild_image( - **FORGE image rebuilds**: Rebuild FORGE container after code changes - **Dependency updates**: Trigger rebuilds when base images change - **Testing**: Validate Build configurations work correctly -- **CI/CD integration**: Automated rebuilds in pipelines \ No newline at end of file +- **CI/CD integration**: Automated rebuilds in pipelines diff --git a/projects/core/tests/README.md b/projects/core/tests/README.md index 44bbe0541..3f7edcc46 100644 --- a/projects/core/tests/README.md +++ b/projects/core/tests/README.md @@ -62,4 +62,4 @@ If tests fail, check: 1. That FORGE_HOME is properly set or discoverable 2. That expected projects exist with toolbox directories 3. That toolbox commands have `main.py` files with `run` functions -4. That project `__init__.py` files have docstrings \ No newline at end of file +4. That project `__init__.py` files have docstrings diff --git a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml index 788170f0d..ba27d5d27 100644 --- a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml +++ b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml @@ -5,4 +5,4 @@ metadata: spec: logLevel: Normal managementState: Managed - operatorLogLevel: Normal \ No newline at end of file + operatorLogLevel: Normal diff --git a/projects/matrix_benchmarking/subproject/PLUGINS.md b/projects/matrix_benchmarking/subproject/PLUGINS.md index 5499f7985..e98a1dc68 100644 --- a/projects/matrix_benchmarking/subproject/PLUGINS.md +++ b/projects/matrix_benchmarking/subproject/PLUGINS.md @@ -155,4 +155,4 @@ When creating a new visualization plugin: 5. Create plotting and analysis components 6. Test with sample data -For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. \ No newline at end of file +For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. diff --git a/projects/mcp_gateway/orchestration/presets.d/presets.yaml b/projects/mcp_gateway/orchestration/presets.d/presets.yaml index 55fe683ba..19c1267c3 100644 --- a/projects/mcp_gateway/orchestration/presets.d/presets.yaml +++ b/projects/mcp_gateway/orchestration/presets.d/presets.yaml @@ -61,4 +61,4 @@ large-scale: experiment.target: [gateway] experiment.duration_seconds: 240 experiment.warmup_seconds: 60 - experiment.tools_per_server: 10 \ No newline at end of file + experiment.tools_per_server: 10 From 2de1912da6fce645a8ec2b7b37666d1b48903009 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 14:26:16 +0200 Subject: [PATCH 0012/1072] [caliper] Factorize the S3 prefix code generation --- .../caliper/cli/orchestration_entrypoints.py | 12 ++--- projects/caliper/cli/s3_export.py | 54 +++++++++++++------ projects/caliper/cli/s3_import.py | 2 +- projects/caliper/orchestration/postprocess.py | 3 -- .../caliper/orchestration/step_logging.py | 9 ++-- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 5e8fba7d4..4e273e4b4 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -76,13 +76,13 @@ def s3_import_entrypoint( s3_config = postprocess_config.s3.import_ # Build import prefix using instance + directory - import_prefix_parts = [] - if s3_parent_config.instance: - import_prefix_parts.append(s3_parent_config.instance) - if s3_parent_config.directory: - import_prefix_parts.append(s3_parent_config.directory) + from projects.caliper.cli.s3_export import build_s3_prefix - import_prefix = "/".join(import_prefix_parts) if import_prefix_parts else "" + import_prefix = build_s3_prefix( + instance=s3_parent_config.instance, + directory=s3_parent_config.directory, + trailing_slash=False, + ) import_dir = output_dir / s3_config.output_dir log_s3_import_command( diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index 299305f17..5c500ea55 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -28,6 +28,38 @@ logger = logging.getLogger(__name__) +def build_s3_prefix( + instance: str | None = None, + directory: str | None = None, + upload_id: str | None = None, + trailing_slash: bool = True, +) -> str: + """Build S3 prefix from components, skipping empty parts. + + Args: + instance: Instance identifier (optional) + directory: Directory identifier (optional) + upload_id: Upload identifier (optional) + trailing_slash: Whether to add trailing slash (default: True) + + Returns: + Formatted S3 prefix path + """ + components = [] + if instance: + components.append(instance) + if directory: + components.append(directory) + if upload_id: + components.append(upload_id) + + if not components: + return "" + + prefix = "/".join(components) + return prefix + "/" if trailing_slash else prefix + + def list_ai_data_files(ai_data_dir: Path) -> list[Path]: """List all files in the AI evaluation export directory. @@ -287,14 +319,9 @@ def run_s3_export_with_explicit_paths( logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") # Construct the full S3 path: {instance}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") @@ -596,14 +623,9 @@ def run_s3_export( logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") # Construct the full S3 path: {instance}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index 5f2d8080a..e591ec4d6 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -221,7 +221,7 @@ def run_s3_import_with_explicit_params( Args: bucket: S3 bucket name - prefix: S3 prefix path (full path including s3.prefix/instance/directory) + prefix: S3 prefix path (full path including instance/directory) output_dir: Path to output directory for downloaded files vault: Vault name containing AWS credentials aws_credentials_file: AWS credentials file within vault diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ee285878e..e8dbe4b3f 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1234,10 +1234,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: # Log the CLI command to reproduce this step s3_parent_config = self.config.s3 - export_config = s3_parent_config.export export_path = f"{s3_parent_config.instance}/{s3_parent_config.directory}" - if export_config.prefix: - export_path += f"/{export_config.prefix}" log_s3_export_command( bucket=s3_parent_config.bucket, diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 474ed72c4..0ef4dc1ba 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -400,12 +400,9 @@ def log_s3_import_command( ) -> None: """Log the CLI command to reproduce the S3 import step.""" # Build prefix from instance and directory - prefix_parts = [] - if instance: - prefix_parts.append(instance) - if directory: - prefix_parts.append(directory) - prefix = "/".join(prefix_parts) if prefix_parts else "" + from projects.caliper.cli.s3_export import build_s3_prefix + + prefix = build_s3_prefix(instance=instance, directory=directory, trailing_slash=False) command_parts = [ f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' From 0fb91e5dc89fa5b31d009b8be173a60d70771190 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:02:11 +0200 Subject: [PATCH 0013/1072] [caliper] orchestration: postprocess: include the log file in the step status --- projects/caliper/orchestration/postprocess.py | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e8dbe4b3f..d840419a7 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -677,8 +677,20 @@ def _setup_paths(self) -> None: self.step_logs_dir = Path(env.ARTIFACT_DIR) self.step_logs_dir.mkdir(parents=True, exist_ok=True) - def _add_step(self, step_name: str, step_data: dict[str, Any]) -> None: + def _add_step( + self, step_name: str, step_data: dict[str, Any], log_file: Path | None = None + ) -> None: """Add a step result to the steps list.""" + if log_file: + # Make log file path relative to artifact root where logs are actually stored + try: + artifact_root = Path(env.ARTIFACT_DIR) + relative_log_path = log_file.relative_to(artifact_root) + step_data["log_file"] = str(relative_log_path) + except ValueError: + # If log file is not under artifact root, use absolute path + step_data["log_file"] = str(log_file) + self.steps.append({step_name: step_data}) def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, Any]) -> bool: @@ -760,7 +772,7 @@ def _build_result(self, final_status: str, test_block: dict[str, Any]) -> dict[s return { "final_status": final_status, "success": success_value, - "base_directory": str(self.visualize_output_dir) if self.visualize_output_dir else None, + "base_directory": str(Path(env.ARTIFACT_DIR)), "test_phase": test_block, "steps": self.steps, } @@ -770,7 +782,7 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - with step_logging("caliper_parse", self.step_logs_dir): + with step_logging("caliper_parse", self.step_logs_dir) as log_file: try: model, mod_str = parse_entrypoint( self.config, @@ -796,6 +808,7 @@ def _run_parse_step(self) -> None: "parse_cache_ref": model.parse_cache_ref, "completed_at": time.time(), }, + log_file, ) except Exception as e: # noqa: BLE001 self.parse_failed = True @@ -808,6 +821,7 @@ def _run_parse_step(self) -> None: "traceback": traceback.format_exc(), "completed_at": time.time(), }, + log_file, ) def _run_visualize_step(self) -> None: @@ -815,7 +829,7 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - with step_logging("caliper_visualize", self.step_logs_dir): + with step_logging("caliper_visualize", self.step_logs_dir) as log_file: try: mod_str, plugin = load_plugin_entrypoint( self.config, tree_root=self.tree_root, manifest_path=self.manifest_path @@ -879,6 +893,7 @@ def _run_visualize_step(self) -> None: "output_files": relative_paths, "completed_at": time.time(), }, + log_file, ) except Exception as e: # noqa: BLE001 @@ -892,6 +907,7 @@ def _run_visualize_step(self) -> None: "traceback": traceback.format_exc(), "completed_at": time.time(), }, + log_file, ) def _run_kpi_and_ai_data_steps(self) -> None: @@ -998,11 +1014,11 @@ def _run_artifacts_to_kpis_step( ) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: - with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir): + with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: result = _run_artifacts_to_kpis( self.config, plugin, model, output_dir, mod_str, self.tree_root ) - self._add_step("artifacts_to_kpis", result) + self._add_step("artifacts_to_kpis", result, log_file) self._check_step_result_and_set_failure("artifacts_to_kpis", result) else: self._add_step( @@ -1027,11 +1043,11 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - with step_logging("caliper_kpis_to_csv", self.step_logs_dir): + with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: # Path to the JSON file for reference in command logging kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv(self.config, plugin, model, output_dir, kpi_json_path) - self._add_step("kpis_to_csv", result) + self._add_step("kpis_to_csv", result, log_file) if result.get("status") == "failed": # CSV export failure doesn't affect overall status - it's supplementary logger.warning("KPI CSV export failed but continuing execution") @@ -1051,12 +1067,12 @@ def _run_artifacts_to_ai_data_step( ) return - with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir): + with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: try: result = _run_artifacts_to_ai_data( self.config, plugin, model, output_dir, mod_str, self.tree_root ) - self._add_step("artifacts_to_ai_data", result) + self._add_step("artifacts_to_ai_data", result, log_file) logger.info(f"AI eval export result: {result}") # Check if the result indicates failure or warning @@ -1065,7 +1081,7 @@ def _run_artifacts_to_ai_data_step( except Exception as e: logger.exception("AI eval export failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("artifacts_to_ai_data", step_result) + self._add_step("artifacts_to_ai_data", step_result, log_file) self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) def _run_s3_import_step(self, output_dir: Path) -> None: @@ -1081,7 +1097,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_import", self.step_logs_dir): + with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: try: result = s3_import_entrypoint(self.config, output_dir) @@ -1092,7 +1108,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: result.pop("import_dir"), output_dir ) - self._add_step("s3_import", result) + self._add_step("s3_import", result, log_file) logger.info(f"S3 import result: {result}") self._check_step_result_and_set_failure("s3_import", result) @@ -1100,7 +1116,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: except Exception as e: logger.exception("S3 import failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("s3_import", step_result) + self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: @@ -1143,7 +1159,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - with step_logging("caliper_analyse_kpis", self.step_logs_dir): + with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: try: # Convert relative path to absolute for analyze function current_kpis_path = output_dir / current_kpis_file @@ -1158,7 +1174,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: result["output_file"], output_dir ) - self._add_step("analyse_kpis", result) + self._add_step("analyse_kpis", result, log_file) logger.info(f"KPI analysis result: {result}") self._check_step_result_and_set_failure("analyse_kpis", result) @@ -1166,7 +1182,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: except Exception as e: logger.exception("KPI analysis failed") step_result = {"status": "failed", "error": str(e)} - self._add_step("analyse_kpis", step_result) + self._add_step("analyse_kpis", step_result, log_file) self._check_step_result_and_set_failure("analyse_kpis", step_result) def _run_s3_export_step(self, output_dir: Path) -> None: @@ -1182,7 +1198,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_export", self.step_logs_dir): + with step_logging("caliper_s3_export", self.step_logs_dir) as log_file: try: # Collect file paths from previous steps kpis_file = None @@ -1253,7 +1269,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ai_data_dir=ai_data_dir, analysis_file=analysis_file, ) - self._add_step("s3_export", result) + self._add_step("s3_export", result, log_file) if result.get("status") == "success": # Format status for better readability @@ -1276,7 +1292,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" # Log programming errors more prominently - if isinstance(e, (AttributeError, NameError, TypeError)): + if isinstance(e, AttributeError | NameError | TypeError): logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") logger.exception("Full traceback for programming error:") else: @@ -1289,7 +1305,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: "exception_type": type(e).__name__, "completed_at": time.time(), } - self._add_step("s3_export", step_result) + self._add_step("s3_export", step_result, log_file) self._check_step_result_and_set_failure("s3_export", step_result) def _compute_final_status(self) -> str: From bd91cac75b61cd8bd1000b7e53255c689d641334 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:02:33 +0200 Subject: [PATCH 0014/1072] [caliper] orchestration: notification: add a link to the step log file --- .../caliper/orchestration/notification.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index c1e8fe46f..8901afdf6 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -27,6 +27,7 @@ class PostprocessStepResult: uploaded_files: int | None = None downloaded_files: int | None = None failed_files: int | None = None + log_file: str | None = None # relative path to step log file @dataclass @@ -76,7 +77,19 @@ def format_postprocess_status_notification( for step_name, step_result in sorted_steps: step_emoji = _get_step_emoji(step_result.status) - lines.append(f"- {step_emoji} **{step_name}**: `{step_result.status}`") + + # Create step name as link to log file if available + if step_result.log_file and get_file_link: + try: + log_url = get_file_link(step_result.log_file) + step_name_display = f"[**{step_name}**]({log_url})" + except Exception: + # Fallback to plain text if link generation fails + step_name_display = f"**{step_name}**" + else: + step_name_display = f"**{step_name}**" + + lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") # Add step message if available (but not for warning/failed steps to avoid duplication) if step_result.message and step_result.status not in ("warning", "failed", "failure"): @@ -208,7 +221,9 @@ def format_postprocess_status_notification( step_result, "paths", None ) if file_paths and get_file_link: - lines.extend(_format_step_file_links(step_name, file_paths, get_file_link)) + lines.extend( + _format_step_file_links(step_name, file_paths, get_file_link, step_result) + ) return "\n".join(lines) if lines else "" @@ -253,7 +268,7 @@ def _get_step_emoji(status: str) -> str: def _format_step_file_links( - step_name: str, file_paths: list[str], get_file_link: callable + step_name: str, file_paths: list[str], get_file_link: callable, step_result=None ) -> list[str]: """Format file paths as clickable links using the provided callback. @@ -261,6 +276,7 @@ def _format_step_file_links( step_name: Name of the step file_paths: List of relative file paths get_file_link: Callback function to generate URLs from file paths + step_result: Step result object containing additional metadata (optional) Returns: List of formatted link strings @@ -270,6 +286,9 @@ def _format_step_file_links( lines = [] + # Check if we need to combine output_dir with file paths (e.g., for visualize step) + output_dir = getattr(step_result, "output_dir", None) if step_result else None + # Group files by type for better organization file_groups = _group_files_by_type(file_paths) @@ -277,7 +296,13 @@ def _format_step_file_links( for file_type, files in file_groups.items(): for file_path in files: try: - file_url = get_file_link(file_path) + # Combine output_dir with file_path if available + if output_dir: + full_path = f"{output_dir}/{file_path}" + else: + full_path = file_path + + file_url = get_file_link(full_path) file_name = _get_display_name(file_path) emoji = "📊" if file_type == "visualization" else "📄" lines.append(f" - {emoji} [{file_name}]({file_url})") @@ -377,6 +402,7 @@ def parse_postprocess_result(status_data: dict) -> PostprocessResult | None: uploaded_files=step_data.get("uploaded_files"), downloaded_files=step_data.get("downloaded_files"), failed_files=step_data.get("failed_files"), + log_file=step_data.get("log_file"), ) return PostprocessResult( From f1ecbed3bde5d16e97cc79cd0159e5e9d8ac508f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:05:16 +0200 Subject: [PATCH 0015/1072] [core] ci_entrypoint: fournos_resolve: s/print/logger.info/ --- projects/core/ci_entrypoint/fournos_resolve.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/projects/core/ci_entrypoint/fournos_resolve.py b/projects/core/ci_entrypoint/fournos_resolve.py index 0d5a39fa1..f864243d8 100644 --- a/projects/core/ci_entrypoint/fournos_resolve.py +++ b/projects/core/ci_entrypoint/fournos_resolve.py @@ -183,11 +183,9 @@ def resolve_fournos_config( # Show the updated spec logger.info("Updated FournosJob spec:") - print("=" * 60) - print("Updated FournosJob spec:") - print("=" * 60) - print(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) - print("=" * 60) + logger.info("=" * 60) + logger.info(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) + logger.info("=" * 60) if dry_run: logger.info("DRY RUN: Not applying changes to cluster") From feea37de957f48501f7cb6ffa0b884de191e2cbc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 15:31:55 +0200 Subject: [PATCH 0016/1072] [skeleton] orchestration: disable the agent by default --- projects/skeleton/orchestration/config.yaml | 2 +- projects/skeleton/orchestration/presets.d/presets.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 91f2a4761..eb9bf99b2 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -105,7 +105,7 @@ caliper: include_ai_data: true agentic: - enabled: true + enabled: false model_key: qwen-3-6-35b on_failure: enabled: true diff --git a/projects/skeleton/orchestration/presets.d/presets.yaml b/projects/skeleton/orchestration/presets.d/presets.yaml index e49eca21f..211f236a8 100644 --- a/projects/skeleton/orchestration/presets.d/presets.yaml +++ b/projects/skeleton/orchestration/presets.d/presets.yaml @@ -16,3 +16,6 @@ medium_test: long_test: skeleton.test.duration_seconds: 300 # Long 5 minute test + +agent: + agentic.enabled: true From 4b2e3633100c037adf9a89bee42fd288b08be1ba Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:10 +0200 Subject: [PATCH 0017/1072] [caliper] cli: s3_import: don't show sample object keys --- projects/caliper/cli/s3_import.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index e591ec4d6..db6349fd8 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -276,12 +276,6 @@ def run_s3_import_with_explicit_params( objects = list_s3_objects(s3_client, bucket, import_s3_prefix, max_downloads) logger.info(f"Found {len(objects)} total objects in s3://{bucket}/{import_s3_prefix}") - if objects: - logger.info("Sample object keys:") - for i, obj in enumerate(objects[:5]): # Show first 5 objects - logger.info(f" {i + 1}: {obj['key']}") - if len(objects) > 5: - logger.info(f" ... and {len(objects) - 5} more objects") # Filter objects based on parameters logger.info( From e0f5b02b4f7f33bb404810ea7e6cb97e54817887 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:40 +0200 Subject: [PATCH 0018/1072] [caliper] orchestration: postprocess: pretty print the results --- projects/caliper/orchestration/postprocess.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index d840419a7..2bfd336b4 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import logging import time import traceback @@ -291,8 +292,6 @@ def _run_artifacts_to_kpis( kpis = plugin.compute_kpis(model) output_file.parent.mkdir(parents=True, exist_ok=True) - import json - # Transform flat KPI list into hierarchical structure hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) @@ -357,8 +356,6 @@ def _run_artifacts_to_ai_data( # Write main AI eval payload output_file.parent.mkdir(parents=True, exist_ok=True) - import json - with open(output_file, "w") as f: json.dump(payload, f, indent=2) @@ -478,7 +475,6 @@ def _export_test_entries_with_artifacts( # Write entry metadata entry_metadata_file = test_entry_dir / "entry_metadata.json" - import json with open(entry_metadata_file, "w") as f: json.dump(entry_info, f, indent=2) @@ -1073,7 +1069,9 @@ def _run_artifacts_to_ai_data_step( self.config, plugin, model, output_dir, mod_str, self.tree_root ) self._add_step("artifacts_to_ai_data", result, log_file) - logger.info(f"AI eval export result: {result}") + + logger.info("AI eval export result:") + logger.info(json.dumps(result, indent=2, default=str)) # Check if the result indicates failure or warning self._check_step_result_and_set_failure("artifacts_to_ai_data", result) @@ -1109,7 +1107,9 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) self._add_step("s3_import", result, log_file) - logger.info(f"S3 import result: {result}") + + logger.info("S3 import result:") + logger.info(json.dumps(result, indent=2, default=str)) self._check_step_result_and_set_failure("s3_import", result) @@ -1175,7 +1175,9 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: ) self._add_step("analyse_kpis", result, log_file) - logger.info(f"KPI analysis result: {result}") + + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) self._check_step_result_and_set_failure("analyse_kpis", result) From 823c81174cb50bb4aa7cbe3318326e3d9541e1c9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:10:53 +0200 Subject: [PATCH 0019/1072] [caliper] orchestration: step_logging: remove the log duplication --- projects/caliper/orchestration/step_logging.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 0ef4dc1ba..85cbb8f36 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -513,21 +513,6 @@ def log_analyze_command( ) -> None: """Log the CLI command to reproduce the analyze step.""" - command = f'caliper kpi analyze --current "{current_kpis_path}" --baseline-dir "{historical_kpis_dir}" --output "{output_path}" --plugin "{plugin_module}"' - - step_args = { - "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, - "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, - "output_path": str(output_path) if output_path else None, - "plugin_module": plugin_module, - } - - _log_command_banner( - "caliper_analyze_kpis", - command, - "Analyze current KPIs against historical data for regression detection", - step_args, - ) # Use the correct command structure: caliper kpi analyze command = "caliper kpi analyze" From 157c76d1a6a7c688a9069577e8080bef309d91ee Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:17:03 +0200 Subject: [PATCH 0020/1072] [llm_d] orchestration: config: save the reports in a dedicated directory --- projects/llm_d/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 36f12e78f..de1e08125 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -48,7 +48,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [] report_group: all visualize_config: projects/guidellm/postprocess/guidellm/visualize-groups.yaml From 1a2330f82bc21b9afe734a05209977cfd77a3abc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:17:16 +0200 Subject: [PATCH 0021/1072] [skeleton] orchestration: config: save the reports in a dedicated directory --- projects/skeleton/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index eb9bf99b2..9ee43c738 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -60,7 +60,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [summary_table, throughput_chart] report_group: null From eccf8448cdc6f909fda8f72fdcaadc6b51e6958d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:18:13 +0200 Subject: [PATCH 0022/1072] [caliper] orchestration: store all the step logs in the log file --- .../caliper/orchestration/step_logging.py | 62 +++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 85cbb8f36..7ef0f6d17 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -105,9 +105,12 @@ def step_logging_indexed( """ Context manager for step-specific logging with explicit step index. - Captures all logging output during the context and writes it to a dedicated + Captures ALL logging output during the context and writes it to a dedicated file named with the pattern: {step_index:03d}__{step_name}.log + This context manager temporarily configures the entire logging system to + ensure that ALL log messages from any logger get captured to the step log file. + Args: step_name: Name of the step (e.g., "caliper_parse", "caliper_visualize") step_index: Zero-based index of the step for ordering (e.g., 0, 1, 2...) @@ -121,9 +124,6 @@ def step_logging_indexed( logger.info("This will go to 000__caliper_parse.log") run_parse_operations() """ - # Ensure the global step handler is attached - _ensure_step_handler() - # Create the log file path log_filename = f"{step_index:03d}__{step_name}.log" log_file = output_dir / log_filename @@ -138,16 +138,55 @@ def step_logging_indexed( # Use same format as console output (no timestamp prefix, just the message) file_handler.setFormatter(logging.Formatter("%(message)s")) - # Store the file handler in thread-local storage - _step_local_handlers.file_handler = file_handler + # Get the root logger to capture everything + root_logger = logging.getLogger() + + # Save original state to restore later + original_handlers = root_logger.handlers.copy() + + # Note: We now use direct root logger capture instead of the StepLocalHandler mechanism + # This provides more reliable and comprehensive log capture try: + # Add our file handler directly to the root logger + root_logger.addHandler(file_handler) + + # Save propagation states and ensure loggers propagate to root + saved_logger_states = {} + + # Get all existing loggers from the logger registry + for logger_name in logging.getLogger().manager.loggerDict: + existing_logger = logging.getLogger(logger_name) + # Save original state (only propagate, don't touch levels) + saved_logger_states[logger_name] = { + "propagate": existing_logger.propagate, + } + # Only ensure propagation, don't change levels + existing_logger.propagate = True + yield log_file + finally: - # Clean up: close and remove the thread-local handler - if hasattr(_step_local_handlers, "file_handler"): - _step_local_handlers.file_handler.close() - del _step_local_handlers.file_handler + # Remove our file handler from root logger + if file_handler in root_logger.handlers: + root_logger.removeHandler(file_handler) + + # Restore original handlers completely + # (Remove any handlers that were added, restore original list) + root_logger.handlers.clear() + root_logger.handlers.extend(original_handlers) + + # Restore original logger propagation states + for logger_name, state in saved_logger_states.items(): + try: + existing_logger = logging.getLogger(logger_name) + existing_logger.propagate = state["propagate"] + except Exception: + # Ignore errors when restoring logger states + pass + + # Close the file handler + file_handler.close() @contextmanager @@ -213,7 +252,8 @@ def _log_command_banner( description: Brief description of what this step does step_args: Dictionary of step arguments for reference """ - logger = logging.getLogger(__name__) + # Use root logger to ensure step handler captures the banner + logger = logging.getLogger() LINE_WIDTH = 80 From c7d8c206923e49638f75961e713c7e75c4ff175b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:43:05 +0200 Subject: [PATCH 0023/1072] [caliper] orchestration: postprocess: visualize: show the output_dir --- projects/caliper/orchestration/postprocess.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2bfd336b4..6a7d5a8f5 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -92,6 +92,7 @@ def _resolve_paths( def _resolve_visualize_output_dir( raw: str | None, + base_dir: Path | None = None, ) -> Path: if raw is None or not str(raw).strip(): raise ValueError( @@ -100,7 +101,18 @@ def _resolve_visualize_output_dir( p = Path(raw).expanduser() if p.is_absolute(): return p.resolve() - raise ValueError("caliper.postprocess.visualize.output_dir must be an absolute path") + + # Handle relative paths by making them relative to base_dir (defaults to env.ARTIFACT_DIR) + if base_dir is None: + base_dir = env.ARTIFACT_DIR + + if base_dir is None: + raise ValueError( + "caliper.postprocess.visualize.output_dir must be an absolute path when no base directory is available" + ) + + resolved_path = (base_dir / p).resolve() + return resolved_path def _resolve_visualize_config_path( @@ -838,9 +850,14 @@ def _run_visualize_step(self) -> None: if self.visualize_output_dir is not None: output_dir = self.visualize_output_dir.expanduser().resolve() + logger.info(f"Using explicit visualize output directory: {output_dir}") else: output_dir = _resolve_visualize_output_dir( self.config.visualize.output_dir, + base_dir=env.ARTIFACT_DIR, + ) + logger.info( + f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" ) # Log command to reproduce this step @@ -881,12 +898,20 @@ def _run_visualize_step(self) -> None: # If path is not under output_dir, keep as-is relative_paths.append(str(path)) + # Calculate relative output_dir path from base_directory + try: + relative_output_dir = str(output_dir.relative_to(env.ARTIFACT_DIR)) + except ValueError: + # If output_dir is not under ARTIFACT_DIR, use absolute path + relative_output_dir = str(output_dir) + self._add_step( "visualize", { "status": "success", "plugin_module": mod_str, "output_files": relative_paths, + "output_dir": relative_output_dir, "completed_at": time.time(), }, log_file, From 0e4a726dfd7fa3af2a5e66c1656e002144ada375 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 16:46:20 +0200 Subject: [PATCH 0024/1072] [skeleton] postprocess: default: make more verbose --- .../default/plotting/summary_table.py | 18 +++++++++++- .../default/plotting/throughput_chart.py | 29 +++++++++++++++++-- .../skeleton/postprocess/default/plugin.py | 19 ++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/projects/skeleton/postprocess/default/plotting/summary_table.py b/projects/skeleton/postprocess/default/plotting/summary_table.py index 8e19ef2a7..e758b5e03 100644 --- a/projects/skeleton/postprocess/default/plotting/summary_table.py +++ b/projects/skeleton/postprocess/default/plotting/summary_table.py @@ -3,10 +3,13 @@ from __future__ import annotations import html as html_lib +import logging from pathlib import Path from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class SummaryTablePlot: """Generates an HTML summary table of test results.""" @@ -23,11 +26,19 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating summary table HTML report") + logger.info(f"Processing {len(model.unified_result_records)} result records") + rows = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): scenario = r.distinguishing_labels.get("scenario", "") tp = r.metrics.get("throughput", "") lat = r.metrics.get("latency_ms", "") + + logger.debug( + f"Row {i}: path={r.test_base_path}, scenario={scenario}, throughput={tp}, latency={lat}" + ) + rows.append( "" f"{html_lib.escape(str(r.test_base_path))}" @@ -37,6 +48,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: "" ) + logger.info(f"Generated {len(rows)} table rows") + table_html = ( "" "Skeleton sample — summary" @@ -50,5 +63,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: ) output_file = output_dir / "summary_table.html" + logger.info(f"Writing summary table to: {output_file}") output_file.write_text(table_html, encoding="utf-8") + file_size = output_file.stat().st_size + logger.info(f"Summary table HTML written successfully ({file_size} bytes)") return str(output_file) diff --git a/projects/skeleton/postprocess/default/plotting/throughput_chart.py b/projects/skeleton/postprocess/default/plotting/throughput_chart.py index 075ccd610..774d92f84 100644 --- a/projects/skeleton/postprocess/default/plotting/throughput_chart.py +++ b/projects/skeleton/postprocess/default/plotting/throughput_chart.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging from pathlib import Path import plotly.graph_objects as go from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class ThroughputChartPlot: """Generates a Plotly bar chart of throughput by scenario.""" @@ -24,21 +27,34 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating throughput chart HTML report") + logger.info( + f"Processing {len(model.unified_result_records)} result records for throughput data" + ) + xs: list[str] = [] ys: list[float] = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): label = str(r.distinguishing_labels.get("scenario") or r.test_base_path) raw = r.metrics.get("throughput", 0) try: y = float(raw) except (TypeError, ValueError): + logger.debug( + f"Record {i}: Could not convert throughput '{raw}' to float, using 0.0" + ) y = 0.0 + logger.debug(f"Record {i}: label='{label}', throughput={y}") xs.append(label) ys.append(y) + logger.info(f"Collected throughput data for {len(xs)} scenarios") + logger.info(f"Throughput range: {min(ys) if ys else 0} - {max(ys) if ys else 0}") + + logger.info("Creating Plotly bar chart") fig = go.Figure(data=[go.Bar(x=xs, y=ys)]) fig.update_layout( title="Throughput by scenario", @@ -49,5 +65,12 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: from projects.caliper.postprocess.helpers.visualization_utils import write_full_page_html output_file = output_dir / "throughput_chart.html" - write_full_page_html(fig, str(output_file), "Throughput Chart") - return str(output_file) + logger.info(f"Writing throughput chart to: {output_file}") + success = write_full_page_html(fig, str(output_file), "Throughput Chart") + if success: + file_size = output_file.stat().st_size + logger.info(f"Throughput chart HTML written successfully ({file_size} bytes)") + return str(output_file) + else: + logger.error(f"Failed to write throughput chart to {output_file}") + return None diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 14fe301b2..04a10eb16 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -15,6 +16,8 @@ from .parsing import SkeletonKpiHandler, SkeletonParser from .plotting import SummaryTablePlot, ThroughputChartPlot +logger = logging.getLogger(__name__) + class SkeletonDefaultPlugin(PostProcessingPlugin): """ @@ -47,16 +50,32 @@ def visualize( visualize_config: dict[str, Any] | None, ) -> list[str]: """Generate visualization reports.""" + logger.info("Starting skeleton plugin visualization") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Requested report IDs: {report_ids}") + logger.info(f"Group ID: {group_id}") + logger.info(f"Available plot types: {list(self.plots.keys())}") + logger.info(f"Model contains {len(model.unified_result_records)} result records") + output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] wanted = frozenset(report_ids or ()) + logger.info(f"Will generate plots for: {sorted(wanted)}") + for report_id in wanted: if report_id in self.plots: + logger.info(f"Generating plot: {report_id}") plot_class = self.plots[report_id] path = plot_class.generate(model, output_dir) + logger.info(f"Generated plot file: {path}") paths.append(path) + else: + logger.warning(f"Requested plot type '{report_id}' not available in this plugin") + logger.info( + f"Skeleton plugin visualization completed. Generated {len(paths)} files: {paths}" + ) return paths def kpi_catalog(self) -> list[dict[str, Any]]: From 530a52c00f39758a3c301b67e9df7a8b5f80671c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:03:48 +0200 Subject: [PATCH 0025/1072] [guidellm] postprocess: guidellm: use the logger --- .../guidellm/csv_export/kpi_csv_exporter.py | 11 +- .../guidellm/plotting/kpi_report.py | 7 +- .../guidellm/plotting/performance_analysis.py | 125 +++++++++--------- 3 files changed, 77 insertions(+), 66 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py index 4ba749ae8..e4e3420af 100644 --- a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py +++ b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +import logging from pathlib import Path from typing import Any @@ -13,6 +14,8 @@ create_csv_rows_from_kpi_record, ) +logger = logging.getLogger(__name__) + class KPICsvExporter: """Export KPI data to CSV format.""" @@ -62,7 +65,7 @@ def export_kpis_to_csv( rows = create_csv_rows_from_kpi_record(record) csv_rows.extend(rows) except Exception as e: - print(f"Warning: Failed to convert KPI record to CSV row: {e}") + logger.info(f"Warning: Failed to convert KPI record to CSV row: {e}") continue # Write CSV file @@ -90,14 +93,14 @@ def export_kpis_to_csv( # Summary info total_written = len(csv_rows) - print(f"Exported {total_written} KPI records to {output_path}") + logger.info(f"Exported {total_written} KPI records to {output_path}") if skipped_2d_count > 0: - print(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") + logger.info(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") # Count 2D KPI expansion total_2d_rows = sum(1 for row in csv_rows if getattr(row, "is_2d", False)) if total_2d_rows > 0: - print(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") + logger.info(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") return str(output_path) diff --git a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py index 98534c9f3..8eaf0007e 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py +++ b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from html import escape from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ create_report_filename, ) +logger = logging.getLogger(__name__) + def generate_kpi_report( records: list[Any], @@ -35,7 +38,7 @@ def generate_kpi_report( # Handle multiple records by creating sections for each test if len(records) > 1: - print(f"📊 Generating multi-test KPI report for {len(records)} test records") + logger.info(f"📊 Generating multi-test KPI report for {len(records)} test records") return _generate_multi_test_kpi_report(records, output_dir, report_title, report_number) # Use the single record to extract test conditions and KPIs @@ -460,7 +463,7 @@ def _generate_multi_test_kpi_report( test_sections = [] for i, record in enumerate(records, 1): - print(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") + logger.info(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") # Extract test condition labels test_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(record) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 549c0783c..d1794e2ef 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -18,6 +19,8 @@ save_figure, ) +logger = logging.getLogger(__name__) + def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: Any = 0.0) -> Any: """ @@ -51,18 +54,18 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data Returns: DataFrame with all benchmark metrics and distinguishing labels """ - print(f"📊 Converting {len(records)} records to analysis dataframe...") + logger.info(f"📊 Converting {len(records)} records to analysis dataframe...") data = [] # Get parameters that vary across all records for legend names - print("🔍 Analyzing parameter variations for meaningful legend names...") + logger.info("🔍 Analyzing parameter variations for meaningful legend names...") varying_params = get_varying_parameters(records) if varying_params: - print(f" Found varying parameters: {', '.join(sorted(varying_params))}") + logger.info(f" Found varying parameters: {', '.join(sorted(varying_params))}") else: - print(" No varying parameters found - using default naming") + logger.info(" No varying parameters found - using default naming") - print("📝 Processing records and extracting metrics...") + logger.info("📝 Processing records and extracting metrics...") for record in records: # Skip records without GuideLLM data or missing benchmarks if not record.run_identity.get("guidellm") or record.metrics.get("no_benchmarks_found"): @@ -77,7 +80,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if has_curves and isinstance(request_rates, list) and len(request_rates) > 0: # New format: expand performance curves into multiple data points - print( + logger.info( f" 🔄 Expanding performance curves for {legend_name} ({len(request_rates)} points)" ) curves = record.metrics.get("performance_curves", {}) @@ -157,7 +160,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) else: # Old format: single data point per record (backward compatibility) - print(f" 📊 Using scalar metrics for {legend_name}") + logger.info(f" 📊 Using scalar metrics for {legend_name}") # Extract scalar request_rate if it's a single value request_rate = ( request_rates[0] @@ -226,14 +229,14 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) if not data: - print("⚠️ No GuideLLM data found in records") + logger.info("⚠️ No GuideLLM data found in records") return pd.DataFrame() - print(f"✅ Successfully processed {len(data)} data points from {len(records)} records") + logger.info(f"✅ Successfully processed {len(data)} data points from {len(records)} records") df = pd.DataFrame(data) # Sort for consistent ordering - print("📋 Organizing data by configuration, concurrency, and request rate...") + logger.info("📋 Organizing data by configuration, concurrency, and request rate...") # Sort and fill any NaN values in numeric columns with 0 for consistent plotting df = df.sort_values( ["test_configuration", "request_concurrency", "request_rate", "rate_point_index"] @@ -259,7 +262,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data ] ) - print( + logger.info( f"🎯 Expanded {total_records} benchmark records into {len(df)} data points across {len(configs)} configurations:" ) for config in configs: @@ -270,7 +273,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if rate_points > 1 else f"{config_data['request_rate'].iloc[0]:.1f} req/s" ) - print(f" • {config}: {rate_points} rate points ({rate_range})") + logger.info(f" • {config}: {rate_points} rate points ({rate_range})") return df @@ -279,11 +282,11 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): """Create throughput scaling scatter plot.""" try: - print("📈 Creating throughput scaling scatter plot...") + logger.info("📈 Creating throughput scaling scatter plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for throughput scaling plot") + logger.info("⚠️ No data available for throughput scaling plot") return None title = f"Request Throughput vs Concurrency by Configuration{title_context}" @@ -314,22 +317,22 @@ def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ Throughput scaling plot created successfully") + logger.info("✅ Throughput scaling plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create throughput scaling plot: {e}") + logger.info(f"❌ Failed to create throughput scaling plot: {e}") return None def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = ""): """Create latency vs throughput scatter plot.""" try: - print("📈 Creating latency vs throughput trade-off plot...") + logger.info("📈 Creating latency vs throughput trade-off plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for latency vs throughput plot") + logger.info("⚠️ No data available for latency vs throughput plot") return None title = f"Latency vs Throughput Trade-off{title_context}" @@ -353,22 +356,22 @@ def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = "") showlegend=True, width=800, height=500, font={"size": 12}, title_font_size=16 ) - print("✅ Latency vs throughput plot created successfully") + logger.info("✅ Latency vs throughput plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create latency vs throughput plot: {e}") + logger.info(f"❌ Failed to create latency vs throughput plot: {e}") return None def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput vs concurrency line plot.""" try: - print("📈 Creating token throughput vs concurrency scaling plot...") + logger.info("📈 Creating token throughput vs concurrency scaling plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for token throughput vs concurrency plot") + logger.info("⚠️ No data available for token throughput vs concurrency plot") return None title = f"Token Throughput vs Concurrency{title_context}
Higher is better" @@ -399,22 +402,22 @@ def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput vs concurrency plot created successfully") + logger.info("✅ Token throughput vs concurrency plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput vs concurrency plot: {e}") + logger.info(f"❌ Failed to create token throughput vs concurrency plot: {e}") return None def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): """Create TTFT analysis line plot.""" try: - print("📈 Creating TTFT (Time to First Token) analysis plot...") + logger.info("📈 Creating TTFT (Time to First Token) analysis plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for TTFT analysis plot") + logger.info("⚠️ No data available for TTFT analysis plot") return None title = f"TTFT vs Concurrency{title_context}
Lower is better" @@ -440,23 +443,23 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ TTFT analysis plot created successfully") + logger.info("✅ TTFT analysis plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create TTFT analysis plot: {e}") + logger.info(f"❌ Failed to create TTFT analysis plot: {e}") return None def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput percentiles plot.""" try: - print("📈 Creating token throughput percentiles distribution plot...") + logger.info("📈 Creating token throughput percentiles distribution plot...") import plotly.express as px import plotly.graph_objects as go if df.empty: - print("⚠️ No data available for token throughput percentiles plot") + logger.info("⚠️ No data available for token throughput percentiles plot") return None title = f"Output Token Throughput Percentiles{title_context}
Higher is better" @@ -465,7 +468,9 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st # Get unique configurations and colors configurations = sorted(df["test_configuration"].unique()) - print(f" Plotting {len(configurations)} configurations with percentile distributions...") + logger.info( + f" Plotting {len(configurations)} configurations with percentile distributions..." + ) available_colors = px.colors.qualitative.Set1 color_map = { config: available_colors[i % len(available_colors)] @@ -480,7 +485,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ("P75", "output_tokens_per_second_p75", {"width": 3, "dash": "dash"}, 0.9), ("P90", "output_tokens_per_second_p90", {"width": 2, "dash": "dashdot"}, 0.8), ] - print(f" Adding {len(percentiles)} percentile lines per configuration...") + logger.info(f" Adding {len(percentiles)} percentile lines per configuration...") for config in configurations: config_df = df[df["test_configuration"] == config].sort_values("request_concurrency") @@ -510,11 +515,11 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput percentiles plot created successfully") + logger.info("✅ Token throughput percentiles plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput percentiles plot: {e}") + logger.info(f"❌ Failed to create token throughput percentiles plot: {e}") return None @@ -565,7 +570,7 @@ def generate_token_throughput_vs_concurrency( report_number: int | None = None, ) -> str | None: """Generate token throughput vs concurrency analysis and save to file.""" - print("\n🚀 Generating token throughput vs concurrency analysis...") + logger.info("\n🚀 Generating token throughput vs concurrency analysis...") df = create_dataframe_from_records(records) if df.empty: return None @@ -645,16 +650,16 @@ def generate_comprehensive_performance_report( # Create report identifier using core utilities display_title = create_report_title_display(report_title, report_number) - print(f"\n🚀 Starting {display_title} generation...") - print("=" * 70) + logger.info(f"\n🚀 Starting {display_title} generation...") + logger.info("=" * 70) df = create_dataframe_from_records(records) if df.empty: - print("❌ No data available for analysis") + logger.info("❌ No data available for analysis") return None - print("\n📊 Generating performance analysis plots...") - print(" This may take a moment as we create high-quality visualizations...") + logger.info("\n📊 Generating performance analysis plots...") + logger.info(" This may take a moment as we create high-quality visualizations...") # Generate all the plots as figures plot_functions = [ @@ -673,17 +678,17 @@ def generate_comprehensive_performance_report( report_dir = output_dir / report_dir_name report_dir.mkdir(exist_ok=True) - print(f"\n📁 Created report directory: {report_dir_name}") + logger.info(f"\n📁 Created report directory: {report_dir_name}") plots_data = [] for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - print(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") + logger.info(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") try: fig = plot_func(df, title_context) if fig: # Save as both PNG and HTML in the report directory filename = plot_name.lower().replace(" ", "_") - print(f"💾 Saving {plot_name} as PNG and HTML...") + logger.info(f"💾 Saving {plot_name} as PNG and HTML...") # Save PNG image width = 900 if "Percentiles" in plot_name else 800 @@ -704,26 +709,26 @@ def generate_comprehensive_performance_report( f"{report_dir_name}/{Path(html_path).name}", # HTML path ) ) - print(f"✅ {plot_name} saved as PNG and HTML") + logger.info(f"✅ {plot_name} saved as PNG and HTML") else: - print(f"⚠️ {plot_name} could not be created (no figure returned)") + logger.info(f"⚠️ {plot_name} could not be created (no figure returned)") except Exception as e: - print(f"❌ Failed to generate {plot_name}: {e}") + logger.info(f"❌ Failed to generate {plot_name}: {e}") - print(f"\n✅ Successfully generated {len(plots_data)} visualizations!") + logger.info(f"\n✅ Successfully generated {len(plots_data)} visualizations!") # Generate summary statistics - print("\n📊 Computing performance statistics and insights...") + logger.info("\n📊 Computing performance statistics and insights...") summary_stats = _generate_performance_summary(df) if summary_stats.get("best_tokens"): best = summary_stats["best_tokens"] - print(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") + logger.info(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") # Create comprehensive HTML report - print("\n📝 Assembling comprehensive HTML report...") - print(" 🔗 Creating report with images linking to interactive HTML versions...") + logger.info("\n📝 Assembling comprehensive HTML report...") + logger.info(" 🔗 Creating report with images linking to interactive HTML versions...") html_content = _create_comprehensive_html_report_with_images( plots_data, summary_stats, title_context, display_title @@ -734,28 +739,28 @@ def generate_comprehensive_performance_report( "performance_analysis", report_number, report_title, "html" ) report_file = output_dir / filename - print(f"\n💾 Writing {display_title} to: {report_file.name}") + logger.info(f"\n💾 Writing {display_title} to: {report_file.name}") report_file.write_text(html_content, encoding="utf-8") file_size_mb = report_file.stat().st_size / (1024 * 1024) - print(f"📄 Report complete! File size: {file_size_mb:.1f}MB") - print("=" * 70) - print(f"🎉 {display_title} ready: {report_file.name}") + logger.info(f"📄 Report complete! File size: {file_size_mb:.1f}MB") + logger.info("=" * 70) + logger.info(f"🎉 {display_title} ready: {report_file.name}") return str(report_file) except Exception as e: - print(f"❌ Failed to generate comprehensive performance report: {e}") + logger.info(f"❌ Failed to generate comprehensive performance report: {e}") return None def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: """Generate performance summary statistics from the dataframe.""" if df.empty: - print("⚠️ No data available for performance summary") + logger.info("⚠️ No data available for performance summary") return {} - print(" 🔍 Analyzing best performers across all metrics...") + logger.info(" 🔍 Analyzing best performers across all metrics...") # Find best performers best_tokens_idx = df["tokens_per_second"].idxmax() @@ -765,7 +770,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: best_ttft_idx = df["ttft_median_ms"].idxmin() # Configuration analysis - print(" 📊 Computing configuration performance rankings...") + logger.info(" 📊 Computing configuration performance rankings...") config_performance = ( df.groupby("test_configuration") .agg( @@ -779,7 +784,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: .sort_values("tokens_per_second", ascending=False) ) - print(f" ✅ Performance analysis complete for {len(config_performance)} configurations") + logger.info(f" ✅ Performance analysis complete for {len(config_performance)} configurations") return { "total_configurations": len(df["test_configuration"].unique()), From 80ccb5e47467f9a568799a6036bce33b9d1733df Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:04:10 +0200 Subject: [PATCH 0026/1072] [caliper] engine: parse: use the logger --- projects/caliper/engine/parse.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index dc0e6cce8..132e4428e 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from projects.caliper.engine.cache import ( @@ -18,6 +19,8 @@ ) from projects.caliper.engine.traverse import discover_test_bases +logger = logging.getLogger(__name__) + # Validation functions no longer needed - using per-test-base caching @@ -98,14 +101,14 @@ def run_parse( if all_warnings and force_report_partial: for w in all_warnings: - print(f"[parse warning] {w}") # noqa: T201 — CLI feedback + logger.warning(f"[parse warning] {w}") # Show parameter matrix if requested if show_parameter_matrix: - print() # noqa: T201 — CLI feedback + logger.info("") matrix_analysis = analyze_parameter_matrix(all_records) matrix_summary = format_parameter_matrix_summary(matrix_analysis) - print(matrix_summary) # noqa: T201 — CLI feedback - print() # noqa: T201 — CLI feedback + logger.info(matrix_summary) + logger.info("") return model From 9d4c7a7b2b4b78eae0be589c04c412dfb73652f5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:18:29 +0200 Subject: [PATCH 0027/1072] [caliper] relocate kpi_to_csv to the right location --- .../caliper/cli/orchestration_entrypoints.py | 17 +++++++ projects/caliper/engine/kpi/csv_export.py | 43 +++++++++++++++++ projects/caliper/engine/model.py | 46 +++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 10 ++-- 4 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 projects/caliper/engine/kpi/csv_export.py diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 4e273e4b4..9d2481688 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -175,3 +175,20 @@ def analyze_kpis_entrypoint( output_dir=output_dir, current_kpis_file=current_kpis_file, ) + + +def export_kpis_to_csv_entrypoint( + plugin, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, +) -> str: + """Entrypoint for orchestration to export KPIs to CSV.""" + from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv + + return export_kpis_to_csv( + plugin=plugin, + kpi_records=kpi_records, + output_path=output_path, + include_header_comments=include_header_comments, + ) diff --git a/projects/caliper/engine/kpi/csv_export.py b/projects/caliper/engine/kpi/csv_export.py new file mode 100644 index 000000000..baf9ef032 --- /dev/null +++ b/projects/caliper/engine/kpi/csv_export.py @@ -0,0 +1,43 @@ +"""Export KPIs to CSV format using plugin interface.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +def export_kpis_to_csv( + *, + plugin: object, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, +) -> str: + """ + Export KPI records to CSV format using the plugin's CSV export method. + + Args: + plugin: PostProcessingPlugin instance with export_kpis_to_csv method + kpi_records: List of KPI record dictionaries + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file as string + + Raises: + AttributeError: If plugin doesn't have export_kpis_to_csv method + """ + if not hasattr(plugin, "export_kpis_to_csv"): + raise AttributeError( + f"Plugin {plugin.__class__.__name__} does not implement export_kpis_to_csv method" + ) + + # Delegate to plugin-specific CSV export implementation + result_path = plugin.export_kpis_to_csv( + kpi_records=kpi_records, + output_path=output_path, + include_header_comments=include_header_comments, + ) + + return result_path diff --git a/projects/caliper/engine/model.py b/projects/caliper/engine/model.py index 3a786f69c..10fc42b04 100644 --- a/projects/caliper/engine/model.py +++ b/projects/caliper/engine/model.py @@ -105,6 +105,52 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Return canonical-shaped KPI dicts (pre-validated).""" return [] + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + # Default implementation: basic CSV export + import csv + + # Ensure parent directory exists before any file operations + output_path.parent.mkdir(parents=True, exist_ok=True) + + if not kpi_records: + output_path.touch() # Create empty file + return str(output_path) + + with open(output_path, "w", newline="", encoding="utf-8") as csvfile: + if include_header_comments: + csvfile.write("# KPI Export\n") + csvfile.write("# Generated by Caliper\n") + + if kpi_records: + # Get all possible fieldnames from all records + fieldnames = set() + for record in kpi_records: + fieldnames.update(record.keys()) + fieldnames = sorted(fieldnames) + + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for record in kpi_records: + writer.writerow(record) + + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Structured JSON for AI agent evaluation.""" return {"schema_version": "1", "run_id": "", "metrics": {}} diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 6a7d5a8f5..b721c045a 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -20,6 +20,7 @@ from projects.caliper.cli.orchestration_entrypoints import ( analyze_kpis_entrypoint, + export_kpis_to_csv_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, @@ -528,11 +529,10 @@ def _run_kpis_to_csv( output_path=output_file, ) - # Import and use the CSV exporter - from projects.guidellm.postprocess.guidellm.csv_export import quick_export_kpis_to_csv - - result_path = quick_export_kpis_to_csv( - records=kpi_records, + # Use Caliper entrypoint for CSV export + result_path = export_kpis_to_csv_entrypoint( + plugin=plugin, + kpi_records=kpi_records, output_path=output_file, include_header_comments=postprocess_config.kpi.kpis_to_csv.include_header_comments, ) From 7867d9dc07f35232fe468cf8c65376f6b2e4c99b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 17:18:58 +0200 Subject: [PATCH 0028/1072] [guidellm] postprocess: guidellm: plugin: expose the export_kpis_to_csv method --- .../guidellm/postprocess/guidellm/plugin.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index dfe8ff3ed..3e4c775c1 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -175,6 +175,27 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Compute KPI values from the unified model.""" return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format using GuideLLM's CSV exporter. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + from .csv_export import KPICsvExporter + + exporter = KPICsvExporter() + return exporter.export_kpis_to_csv(kpi_records, output_path, include_header_comments) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Build AI evaluation payload from the unified model.""" return self.ai_evaluator.build_payload(model, self) From 7c7137535f4b0277f68c49a7be70b508d7a07b45 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 22:22:42 +0200 Subject: [PATCH 0029/1072] [core] library: replot: show the correct artifacts dir --- projects/caliper/orchestration/postprocess.py | 34 ++++++------------- projects/caliper/orchestration/replot.py | 18 +++++----- projects/core/library/replot.py | 9 +++-- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index b721c045a..2c795f2fc 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -93,27 +93,18 @@ def _resolve_paths( def _resolve_visualize_output_dir( raw: str | None, - base_dir: Path | None = None, ) -> Path: if raw is None or not str(raw).strip(): - raise ValueError( - "caliper.postprocess.visualize.output_dir is required when no explicit visualize_output_directory is provided" - ) + # If empty, use env.ARTIFACT_DIR + return env.ARTIFACT_DIR + p = Path(raw).expanduser() if p.is_absolute(): + # If absolute, don't touch return p.resolve() - - # Handle relative paths by making them relative to base_dir (defaults to env.ARTIFACT_DIR) - if base_dir is None: - base_dir = env.ARTIFACT_DIR - - if base_dir is None: - raise ValueError( - "caliper.postprocess.visualize.output_dir must be an absolute path when no base directory is available" - ) - - resolved_path = (base_dir / p).resolve() - return resolved_path + else: + # If relative, use env.ARTIFACT_DIR / output_dir + return (env.ARTIFACT_DIR / p).resolve() def _resolve_visualize_config_path( @@ -854,7 +845,6 @@ def _run_visualize_step(self) -> None: else: output_dir = _resolve_visualize_output_dir( self.config.visualize.output_dir, - base_dir=env.ARTIFACT_DIR, ) logger.info( f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" @@ -937,12 +927,10 @@ def _run_kpi_and_ai_data_steps(self) -> None: return try: - # Determine output directory - if self.config.visualize.enabled and self.visualize_output_dir: - output_dir = Path(self.visualize_output_dir) - else: - output_dir = Path(self.artifacts_dir) / "postprocess_output" - output_dir.mkdir(parents=True, exist_ok=True) + # Determine output directory for KPI/AI data steps - use base artifact directory + output_dir = env.ARTIFACT_DIR + logger.info(f"KPI steps using base artifact directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) # Load plugin and model model, mod_str = parse_entrypoint( diff --git a/projects/caliper/orchestration/replot.py b/projects/caliper/orchestration/replot.py index 8ff5f6f69..86b6382a4 100644 --- a/projects/caliper/orchestration/replot.py +++ b/projects/caliper/orchestration/replot.py @@ -13,6 +13,7 @@ from projects.caliper.orchestration.postprocess import run_postprocess_from_orchestration_config from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome +from projects.core.library import env from projects.core.library import vault as vault_lib logger = logging.getLogger(__name__) @@ -304,15 +305,14 @@ def run_replot_from_orchestration_config( if artifact_directory.exists(): logger.info("Running post-processing...") - visualize_output_dir = artifact_directory / "postprocess_output" - visualize_output_dir.mkdir(parents=True, exist_ok=True) - - postprocess_result = run_postprocess_from_orchestration_config( - postprocess_config_raw=postprocess_config or {}, - artifacts_dir=artifact_directory, - visualize_output_dir=visualize_output_dir, - test_outcome=TestPhaseOutcome("SUCCESS"), - ) + with env.TempArtifactDir(artifact_directory / "postprocess_output"): + # Let Caliper handle output directory resolution from config + postprocess_result = run_postprocess_from_orchestration_config( + postprocess_config_raw=postprocess_config or {}, + artifacts_dir=artifact_directory, + visualize_output_dir=None, # Let Caliper resolve from config + test_outcome=TestPhaseOutcome("SUCCESS"), + ) else: logger.info("Artifacts directory not found") postprocess_result = { diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index b2e3cd7fb..81696785b 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -167,9 +167,12 @@ def caliper_replot_entrypoint( logger.info("Replot status:") logger.info(yaml.dump(status, indent=2)) - # Show where artifacts have been saved - final_artifact_dir = artifact_directory or env.ARTIFACT_DIR - logger.info(f"📁 Artifacts saved to: {final_artifact_dir}") + try: + logger.info( + f"📁 Artifacts saved to: {status['replot']['postprocess_result']['base_directory']}" + ) + except Exception: + pass # Check if replot was successful replot_status = status.get("replot", {}).get("status", "unknown") From 229cbfd1715b87d8996fdd94cec198a28f9b4343 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 9 Jul 2026 22:57:46 +0200 Subject: [PATCH 0030/1072] [core] library: postprocess: use the right directories --- projects/core/library/postprocess.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index c6ab3186f..893c9e7f9 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -278,16 +278,13 @@ def run_postprocess_after_test( artifact_root_path = Path(artifact_root).resolve() if artifact_root is not None else None with env.NextArtifactDir("postprocessing"): - workspace = Path(env.ARTIFACT_DIR).resolve() logger.info( - "Running Caliper postprocess (artifacts=%s, workspace=%s, test_phase=%s)", + "Running Caliper postprocess (artifacts=%s, test_phase=%s)", artifact_root_path, - workspace, test_outcome.phase if test_outcome else "SUCCESS", ) status = run_orchestration_postprocess( artifact_dir=artifact_root_path, - visualize_output_dir=workspace, test_outcome=test_outcome, ) logger.info( @@ -415,11 +412,13 @@ def run_orchestration_postprocess( def postprocess_command(_ctx, artifact_dir: Path, output_dir: Path): """Run the post-processing pipeline.""" - status = run_orchestration_postprocess( - artifact_dir=artifact_dir, - test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), - visualize_output_dir=output_dir, - ) + # Set ARTIFACT_DIR temporarily so config resolution works with user's output directory + with env.TempArtifactDir(output_dir): + status = run_orchestration_postprocess( + artifact_dir=artifact_dir, + test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), + visualize_output_dir=None, # Let config resolution handle it + ) logger.info("Caliper postprocess status:\n" + yaml.dump(status, indent=2)) # Check success flag and return appropriate exit code From 6e5a351878961c2a4e5f776020d3d06503410608 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 07:28:25 +0200 Subject: [PATCH 0031/1072] [caliper] cli: commands: fix the s3-import command --- projects/caliper/cli/commands.py | 36 +++++++------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index f06881acf..5803dae0a 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -7,7 +7,7 @@ import click -from projects.caliper.cli.s3_import import run_s3_import +from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import @@ -446,11 +446,6 @@ def kpi_s3_import( verbose: bool, ) -> None: """Download historical KPI and analysis data from S3.""" - from projects.caliper.orchestration.postprocess_config import ( - CaliperOrchestrationPostprocessConfig, - CaliperOrchestrationS3ImportSection, - CaliperOrchestrationS3Section, - ) from projects.core.library import vault as vault_lib try: @@ -466,34 +461,19 @@ def kpi_s3_import( # Initialize vault system vault_lib.init(vaults=[vault] if vault else []) - # Parse bucket/prefix to extract instance and directory - prefix_parts = prefix.strip("/").split("/") - instance = prefix_parts[0] if len(prefix_parts) > 0 and prefix_parts[0] else None - directory = prefix_parts[1] if len(prefix_parts) > 1 else None - - s3_import_config = CaliperOrchestrationS3ImportSection( - enabled=True, - output_dir=output_dir.name, + # Run S3 import + result = run_s3_import_with_explicit_params( + bucket=bucket, + prefix=prefix, + output_dir=output_dir, + vault=vault, + aws_credentials_file=aws_credentials_file, include_kpis_json=include_kpis_json, include_kpis_csv=include_kpis_csv, include_ai_data=include_ai_data, max_downloads=max_downloads, ) - s3_config = CaliperOrchestrationS3Section( - bucket=bucket, - instance=instance, - directory=directory, - vault=vault, - aws_credentials_file=aws_credentials_file, - **{"import": s3_import_config}, - ) - - config = CaliperOrchestrationPostprocessConfig(s3=s3_config) - - # Run S3 import - result = run_s3_import(config, output_dir.parent) - if result["status"] == "success": click.echo("✅ S3 import completed successfully") downloaded_files = result.get("files", []) From 8aec9f50b26b36938e4e04d3c4ac3b69b72c34f1 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Fri, 10 Jul 2026 08:24:10 +0100 Subject: [PATCH 0032/1072] chore: Run linter Signed-off-by: Alberto Perdomo --- projects/caliper/orchestration/postprocess.py | 2 +- projects/llm_d/orchestration/ci.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ee285878e..c3ba2b234 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1279,7 +1279,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" # Log programming errors more prominently - if isinstance(e, (AttributeError, NameError, TypeError)): + if isinstance(e, AttributeError | NameError | TypeError): logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") logger.exception("Full traceback for programming error:") else: diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index d604f8b02..f8611771c 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -36,6 +36,7 @@ def init(): run.init() config.init(Path(__file__).parent) + def list_vaults() -> list[str]: """List all vaults (includes both mandatory and optional).""" all_vaults = vault.phase_vault_list_all() From 25bf9a330788b507f356c19bcd39fe5ccbfbbfc0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:01:06 +0200 Subject: [PATCH 0033/1072] [caliper] orchestration: postprocess: abort the post-process on parse failure --- projects/caliper/orchestration/postprocess.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2c795f2fc..2ea5152b1 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -646,13 +646,19 @@ def _execute_orchestration(self) -> dict[str, Any]: logger.info("Starting postprocessing steps") self._run_parse_step() logger.info(f"After parse step: parse_failed={self.parse_failed}") - self._run_visualize_step() - logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") - self._run_kpi_and_ai_data_steps() - logger.info( - f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" - ) - logger.info("All postprocessing steps completed") + + # Abort pipeline if parse failed - no point in running other steps without data + if self.parse_failed: + logger.error("Parse step failed - aborting remaining postprocessing steps") + logger.info("Pipeline aborted due to parse failure") + else: + self._run_visualize_step() + logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") + self._run_kpi_and_ai_data_steps() + logger.info( + f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" + ) + logger.info("All postprocessing steps completed") # Compute final status and build result final_status = self._compute_final_status() From 49d4cf255d51505c5313b6e47761e2afb806aee4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:07:35 +0200 Subject: [PATCH 0034/1072] [caliper] cli: commands: include the status file for the parse command --- projects/caliper/cli/commands.py | 47 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 5803dae0a..05e9a72e1 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -107,12 +107,19 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def parse_cmd( ctx: click.Context, no_cache: bool, cache_dir: Path | None, show_matrix: bool, + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -126,6 +133,9 @@ def parse_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status = {"success": False} + try: model = run_parse( base_dir=artifact_root, @@ -134,12 +144,43 @@ def parse_cmd( use_cache=not no_cache, show_parameter_matrix=show_matrix, ) + + status.update( + { + "success": True, + "plugin_module": mod, + "parsed_records": len(model.unified_result_records), + "cache_ref": str(model.parse_cache_ref) if model.parse_cache_ref else None, + } + ) + + click.echo( + f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" + ) + except Exception as e: # noqa: BLE001 + status.update( + { + "success": False, + "error": str(e), + } + ) click.echo(f"parse failed: {e}", err=True) + + # Write status file if requested + if status_file: + import yaml + + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) + + # Exit with non-zero code on failure + if not status["success"]: sys.exit(2) - click.echo( - f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" - ) @click.command("visualize") From 176cb9af96d3d10d47c325fcf0f8711ac5ef47fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:11:19 +0200 Subject: [PATCH 0035/1072] [caliper] orchestration: launch caliper as an external process --- projects/caliper/cli/commands.py | 8 +- projects/caliper/orchestration/cli_builder.py | 334 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 95 +++-- 3 files changed, 411 insertions(+), 26 deletions(-) create mode 100644 projects/caliper/orchestration/cli_builder.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 05e9a72e1..b54eb72e7 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -77,10 +77,10 @@ def _plugin_tuple(ctx: click.Context): root = _root_obj(ctx) try: - mod = resolve_plugin_module_string( - root.get("plugin_cli"), - root.get("postprocess_config"), - root.get("base_dir"), + mod, _manifest_path = resolve_plugin_module_string( + base_dir=root.get("base_dir"), + postprocess_config=root.get("postprocess_config"), + cli_plugin=root.get("plugin_cli"), ) if not mod: raise RuntimeError("Plugin module not found") diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py new file mode 100644 index 000000000..1563631de --- /dev/null +++ b/projects/caliper/orchestration/cli_builder.py @@ -0,0 +1,334 @@ +"""CLI command builder for Caliper orchestration fork/exec. + +Converts orchestration config objects to CLI argument lists for subprocess execution. +""" + +from __future__ import annotations + +import logging +import subprocess +import sys +from pathlib import Path + +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) + +logger = logging.getLogger(__name__) + + +def build_parse_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper parse. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "parse", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Parse-specific options + if not use_cache: + cmd.append("--no-cache") + + # Enable detailed parameter matrix output + cmd.append("--show-matrix") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_visualize_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_dir: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper visualize. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_dir: Output directory for visualization files + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "visualize", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Visualize-specific options + cmd.extend(["--output-dir", str(output_dir)]) + + if config.visualize.reports: + cmd.extend(["--reports", ",".join(config.visualize.reports)]) + + if config.visualize.report_group: + cmd.extend(["--report-group", config.visualize.report_group]) + + if config.visualize.visualize_config: + cmd.extend(["--visualize-config", str(config.visualize.visualize_config)]) + + # Include/exclude labels + for label in config.visualize.include_labels: + cmd.extend(["--include-label", label]) + + for label in config.visualize.exclude_labels: + cmd.extend(["--exclude-label", label]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def save_command_script(command: list[str], script_path: Path) -> None: + """Save executed command to a shell script for debugging. + + Args: + command: Command arguments list + script_path: Where to save the script + """ + script_path.parent.mkdir(parents=True, exist_ok=True) + + # Convert command to a readable shell script + script_content = "#!/bin/bash\n" + script_content += "# Generated command for debugging\n" + script_content += "# Run this command to reproduce the step manually\n\n" + + # Format command with argument pairs on separate lines + formatted_lines = _format_command_for_script(command) + script_content += formatted_lines + "\n" + + with open(script_path, "w", encoding="utf-8") as f: + f.write(script_content) + + # Make executable + script_path.chmod(0o755) + + +def _format_command_for_script(command: list[str]) -> str: + """Format command for script file with argument pairs on separate lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for shell script + """ + if not command: + return "" + + def escape_arg(arg: str) -> str: + """Escape shell arguments that need quoting.""" + if " " in arg or any(c in arg for c in ["$", "`", '"', "'"]): + return f'"{arg}"' + return arg + + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(escape_arg(command[i])) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) with line continuations + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f" {escape_arg(command[i])} {escape_arg(command[i + 1])}") + i += 2 + else: + # Boolean flag + lines.append(f" {escape_arg(command[i])}") + i += 1 + else: + # Standalone argument + lines.append(f" {escape_arg(command[i])}") + i += 1 + + # Join with line continuations + if len(lines) == 1: + return lines[0] + else: + return lines[0] + " \\\n" + " \\\n".join(lines[1:]) + + +def log_caliper_start_banner(command: list[str], script_path: Path, step_name: str) -> None: + """Log start banner and save command script for Caliper execution. + + Args: + command: Command arguments list + script_path: Where to save the executable script + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + """ + # Save command script for debugging + save_command_script(command, script_path) + + # Format command with argument pairs on new lines + formatted_command = _format_command_for_display(command) + + # Log banner indicating Caliper is starting + logger.info("=" * 60) + logger.info("🚀 STARTING CALIPER %s (fork/exec)", step_name.upper()) + logger.info("📄 Command:") + for line in formatted_command.splitlines(): + logger.info(" %s", line) + logger.info("=" * 60) + + +def _format_command_for_display(command: list[str]) -> str: + """Format command for readable display with argument pairs on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string + """ + if not command: + return "" + + # Start with base command (python -m module subcommand) + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return "\n".join(lines) + + +def handle_caliper_output_and_completion( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Write output to log file + with open(log_file, "w", encoding="utf-8") as log_f: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + status_yaml = yaml.dump(status_data, default_flow_style=False, sort_keys=False) + for line in status_yaml.strip().splitlines(): + logger.info(" %s", line) + + logger.info("=" * 60) + + return status_data diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 2ea5152b1..36b64b230 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -28,6 +28,11 @@ s3_export_entrypoint, s3_import_entrypoint, ) +from projects.caliper.orchestration.cli_builder import ( + build_parse_command, + handle_caliper_output_and_completion, + log_caliper_start_banner, +) from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) @@ -787,48 +792,94 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - with step_logging("caliper_parse", self.step_logs_dir) as log_file: - try: - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, - ) + import subprocess + import tempfile - # Log command to reproduce this step - log_parse_command( - base_dir=self.tree_root, - plugin_module=mod_str, - use_cache=not self.config.parse.no_cache, - manifest_path=self.manifest_path, - ) + import yaml + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + use_cache=not self.config.parse.no_cache, + ) + + # Log start banner and save command script + script_path = env.ARTIFACT_DIR / "parse.sh" + log_caliper_start_banner(command, script_path, "parse") + + # Create log file path for stdout/stderr capture with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + step_index = _get_next_step_index(self.step_logs_dir) + log_file = self.step_logs_dir / f"{step_index:03d}__caliper_parse.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Execute CLI command with output capture + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion(result, log_file, status_file, "parse") + + if result.returncode == 0 and status_data and status_data.get("success", False): self._add_step( "parse", { "status": "success", - "plugin_module": mod_str, - "record_count": len(model.unified_result_records), - "parse_cache_ref": model.parse_cache_ref, + "plugin_module": status_data.get("plugin_module", "unknown"), + "record_count": status_data.get("parsed_records", 0), + "parse_cache_ref": status_data.get("cache_ref"), "completed_at": time.time(), }, log_file, ) - except Exception as e: # noqa: BLE001 + else: self.parse_failed = True - logger.exception("Caliper parse failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper parse failed: %s", error_msg) self._add_step( "parse", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, log_file, ) + except Exception as e: # noqa: BLE001 + self.parse_failed = True + logger.exception("Parse step execution failed") + self._add_step( + "parse", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_visualize_step(self) -> None: """Execute the visualize step if enabled.""" if not self.config.visualize.enabled: From 67a7b84528d239b455d935fc5639294edf807380 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:34:52 +0200 Subject: [PATCH 0036/1072] [caliper] cli: main: enable the logging on CLI command start --- projects/caliper/cli/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 167b32dc3..1bcb875d6 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import sys from pathlib import Path from urllib.parse import parse_qs, urlparse @@ -159,6 +160,13 @@ def artifacts_group(ctx: click.Context) -> None: def run_cli() -> None: """Invoke CLI; on missing required options, print subcommand help.""" + # Configure logging to capture INFO level messages for detailed output + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + force=True, # Override any existing configuration + ) + try: # standalone_mode=False returns exit codes instead of calling sys.exit; # propagate them so failures are non-zero (e.g. ctx.exit(1) from _exit_with_help). From 2684f205aab78d8fb73ae3d3b90bf6c21252821b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:50:20 +0200 Subject: [PATCH 0037/1072] [caliper] cli: commands: include the status file for the visualize command --- projects/caliper/cli/commands.py | 59 ++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index b54eb72e7..504427799 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -6,6 +6,7 @@ from pathlib import Path import click +import yaml from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export @@ -169,8 +170,6 @@ def parse_cmd( # Write status file if requested if status_file: - import yaml - try: status_file.parent.mkdir(parents=True, exist_ok=True) with open(status_file, "w", encoding="utf-8") as f: @@ -195,6 +194,12 @@ def parse_cmd( type=click.Path(path_type=Path), required=True, ) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def visualize_cmd( ctx: click.Context, @@ -204,6 +209,7 @@ def visualize_cmd( include_label: tuple[str, ...], exclude_label: tuple[str, ...], output_dir: Path, + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -217,6 +223,9 @@ def visualize_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status = {"success": False} + try: paths = run_visualize( base_dir=artifact_root, @@ -231,11 +240,51 @@ def visualize_cmd( use_cache=True, cache_path=None, ) + + status.update( + { + "success": True, + "plugin_module": mod, + "output_files": paths, + "output_dir": str(output_dir), + "generated_files": len(paths), + } + ) + + click.echo("Wrote: " + ", ".join(paths)) + except Exception as e: # noqa: BLE001 + import traceback + + error_details = { + "error": str(e), + "traceback": traceback.format_exc(), + } + + status.update( + { + "success": False, + **error_details, + } + ) + + # Print full traceback to stderr for visibility click.echo(f"visualize failed: {e}", err=True) - sys.exit(2) + click.echo("Full traceback:", err=True) + click.echo(traceback.format_exc(), err=True) - click.echo("Wrote: " + ", ".join(paths)) + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) + + # Exit with non-zero code on failure + if not status["success"]: + sys.exit(2) @click.command("list-reports") @@ -439,8 +488,6 @@ def kpi_analyze( result_data = json.load(f) # Convert to YAML for better readability - import yaml - yaml_output = yaml.dump(result_data, default_flow_style=False, indent=2) click.echo("\n📊 Analysis Results:") click.echo("=" * 50) From 9645f715bbfb0d861dfbc421b219236119f4b555 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 10:53:05 +0200 Subject: [PATCH 0038/1072] [caliper] orchestration: launch caliper visualize as an external process --- projects/caliper/orchestration/cli_builder.py | 11 +- projects/caliper/orchestration/postprocess.py | 149 +++++++++++------- 2 files changed, 100 insertions(+), 60 deletions(-) diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 1563631de..0e479c86a 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -112,7 +112,16 @@ def build_visualize_command( cmd.extend(["--report-group", config.visualize.report_group]) if config.visualize.visualize_config: - cmd.extend(["--visualize-config", str(config.visualize.visualize_config)]) + # Resolve the visualize config path properly (same logic as _resolve_visualize_config_path) + viz_config_path = config.visualize.visualize_config + if viz_config_path: + from projects.core.library import env + + viz_path = Path(viz_config_path).expanduser() + if not viz_path.is_absolute(): + # If relative, resolve relative to FORGE_HOME like the original does + viz_path = env.FORGE_HOME / viz_path + cmd.extend(["--visualize-config", str(viz_path.resolve())]) # Include/exclude labels for label in config.visualize.include_labels: diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 36b64b230..aeffc74eb 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -24,12 +24,12 @@ get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, - run_visualize_entrypoint, s3_export_entrypoint, s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( build_parse_command, + build_visualize_command, handle_caliper_output_and_completion, log_caliper_start_banner, ) @@ -47,9 +47,7 @@ log_ai_data_command, log_artifacts_to_kpis_command, log_kpis_to_csv_command, - log_parse_command, log_s3_export_command, - log_visualize_command, step_logging, ) from projects.core.library import env @@ -795,8 +793,6 @@ def _run_parse_step(self) -> None: import subprocess import tempfile - import yaml - # Create status file for CLI output with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: status_file = Path(status_f.name) @@ -831,7 +827,9 @@ def _run_parse_step(self) -> None: ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, "parse") + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, "parse" + ) if result.returncode == 0 and status_data and status_data.get("success", False): self._add_step( @@ -885,62 +883,74 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - with step_logging("caliper_visualize", self.step_logs_dir) as log_file: - try: - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path - ) + import subprocess + import tempfile - viz_cfg_path = _resolve_visualize_config_path( - self.config.visualize.visualize_config, - artifact_tree=self.tree_root, + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Resolve visualize output directory + if self.visualize_output_dir is not None: + output_dir = self.visualize_output_dir.expanduser().resolve() + logger.info(f"Using explicit visualize output directory: {output_dir}") + else: + output_dir = _resolve_visualize_output_dir( + self.config.visualize.output_dir, + ) + logger.info( + f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" ) - if self.visualize_output_dir is not None: - output_dir = self.visualize_output_dir.expanduser().resolve() - logger.info(f"Using explicit visualize output directory: {output_dir}") - else: - output_dir = _resolve_visualize_output_dir( - self.config.visualize.output_dir, - ) - logger.info( - f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" - ) + # Build CLI command + command = build_visualize_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not self.config.parse.no_cache, + ) - # Log command to reproduce this step - log_visualize_command( - base_dir=self.tree_root, - plugin_module=mod_str, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, - ) + # Log start banner and save command script + script_path = env.ARTIFACT_DIR / "visualize.sh" + log_caliper_start_banner(command, script_path, "visualize") - paths = run_visualize_entrypoint( - base_dir=self.tree_root, - plugin_module=mod_str, - plugin=plugin, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, - cache_path=self.cache_path, - ) + # Create log file path for stdout/stderr capture with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + + step_index = _get_next_step_index(self.step_logs_dir) + log_file = self.step_logs_dir / f"{step_index:03d}__caliper_visualize.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Execute CLI command with output capture + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, "visualize" + ) - # Convert paths to relative paths from output_dir + if result.returncode == 0 and status_data and status_data.get("success", False): + # Get output files and paths from status + output_files = status_data.get("output_files", []) + + # Convert absolute paths to relative paths from output_dir if needed relative_paths = [] - for path in paths: + for path in output_files: try: path_obj = Path(path) - relative_path = path_obj.relative_to(output_dir) - relative_paths.append(str(relative_path)) + if path_obj.is_absolute(): + relative_path = path_obj.relative_to(output_dir) + relative_paths.append(str(relative_path)) + else: + relative_paths.append(str(path)) except ValueError: # If path is not under output_dir, keep as-is relative_paths.append(str(path)) @@ -956,28 +966,49 @@ def _run_visualize_step(self) -> None: "visualize", { "status": "success", - "plugin_module": mod_str, + "plugin_module": status_data.get("plugin_module", "unknown"), "output_files": relative_paths, "output_dir": relative_output_dir, + "generated_files": status_data.get("generated_files", len(relative_paths)), "completed_at": time.time(), }, log_file, ) - - except Exception as e: # noqa: BLE001 + else: self.visualize_failed = True - logger.exception("Caliper visualize failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper visualize failed: %s", error_msg) self._add_step( "visualize", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, log_file, ) + except Exception as e: # noqa: BLE001 + self.visualize_failed = True + logger.exception("Visualize step execution failed") + self._add_step( + "visualize", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_kpi_and_ai_data_steps(self) -> None: """Execute KPI generation, CSV export, KPI export, and AI evaluation steps.""" if not self.config.kpi.enabled: From 811e50f97fc2c91a45559a376ee94746fac37c04 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:11:56 +0200 Subject: [PATCH 0039/1072] [caliper] cli: commands: include the status file for the kpi_generate_entrypoint command --- projects/caliper/cli/commands.py | 37 ++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 504427799..a20358eff 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -373,6 +373,9 @@ def ai_eval_export( @click.command("generate") @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_generate( ctx: click.Context, @@ -380,6 +383,7 @@ def kpi_generate( artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -389,14 +393,43 @@ def kpi_generate( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: run_kpi_generate( - base_dir=artifact_root, plugin_module=mod, plugin=plugin, output_path=output + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, ) + status_data = {"success": True, "output_file": str(output)} + click.echo(f"Generated {output}") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Generated {output}") @click.command("import") From 676e28bd80d3d403089470193e93c134c1dbbd93 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:13:33 +0200 Subject: [PATCH 0040/1072] [caliper] orchestration: launch caliper artifacts_to_kpis as an external process --- .../caliper/cli/orchestration_entrypoints.py | 19 +++ projects/caliper/orchestration/cli_builder.py | 45 ++++++++ projects/caliper/orchestration/postprocess.py | 108 ++++++++++++++---- 3 files changed, 150 insertions(+), 22 deletions(-) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index 9d2481688..cde8b2e86 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -177,6 +177,25 @@ def analyze_kpis_entrypoint( ) +def kpi_generate_entrypoint( + base_dir: Path, + plugin_module: str, + plugin: Any, + output: Path, +) -> None: + """Entrypoint for orchestration to generate KPIs.""" + from projects.caliper.engine.kpi.generate import run_kpi_generate + + return run_kpi_generate( + base_dir=base_dir, + plugin_module=plugin_module, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, + ) + + def export_kpis_to_csv_entrypoint( plugin, kpi_records: list[dict[str, Any]], diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 0e479c86a..fd5bcdea6 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -65,6 +65,51 @@ def build_parse_command( return cmd +def build_kpi_generate_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi generate. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output file for KPI data + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "generate", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Generate-specific options + cmd.extend(["--output", str(output_file)]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def build_visualize_command( config: CaliperOrchestrationPostprocessConfig, tree_root: Path, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index aeffc74eb..e6d9d8088 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -11,6 +11,7 @@ import json import logging +import subprocess import time import traceback from pathlib import Path @@ -28,6 +29,7 @@ s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( + build_kpi_generate_command, build_parse_command, build_visualize_command, handle_caliper_output_and_completion, @@ -45,7 +47,6 @@ from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, log_ai_data_command, - log_artifacts_to_kpis_command, log_kpis_to_csv_command, log_s3_export_command, step_logging, @@ -55,6 +56,42 @@ logger = logging.getLogger(__name__) +def _execute_caliper_command( + command: list[str], + step_name: str, + script_file: Path, + log_file: Path, + status_file: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any]]: + """Execute a Caliper CLI command and return result and status data. + + Args: + command: CLI command arguments list + step_name: Name for logging banners (e.g., "PARSE", "VISUALIZE") + script_file: Path to save debug script + log_file: Path to save command output + status_file: Path to read status YAML from + + Returns: + Tuple of (subprocess result, parsed status data) + """ + # Log start banner and save script + log_caliper_start_banner(command, script_file, step_name) + + # Execute command with combined stdout/stderr + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name) + + return result, status_data + + def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. @@ -273,8 +310,9 @@ def _run_artifacts_to_kpis( output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, ) -> dict[str, Any]: - """Generate KPI JSON using the plugin's compute_kpis method.""" + """Generate KPI JSON using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} @@ -286,36 +324,62 @@ def _run_artifacts_to_kpis( } try: - # Write KPI JSON + # Prepare paths output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_artifacts_to_kpis_command( - base_dir=base_dir, - plugin_module=plugin_module, + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Create log file + log_file = output_dir / "000__caliper_kpi_generate.log" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, ) - kpis = plugin.compute_kpis(model) - output_file.parent.mkdir(parents=True, exist_ok=True) + # Execute command using generic function + script_file = env.ARTIFACT_DIR / "kpi_generate.sh" + result, status_data = _execute_caliper_command( + command=command, + step_name="KPI GENERATE", + script_file=script_file, + log_file=log_file, + status_file=status_file, + ) - # Transform flat KPI list into hierarchical structure - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - with open(output_file, "w") as f: - json.dump(hierarchical_data, f, indent=2) + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, output_dir), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } - logger.info( - f"Generated hierarchical KPI structure with {len(kpis)} total records in {output_file}" - ) - return { - "status": "success", - "kpi_count": len(kpis), - "output_file": _make_path_relative_to_base(output_file, output_dir), - "completed_at": time.time(), - } except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} From 9f1857eb6693401bb964d0fd5c5f65ea5bf9fc9a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:14:39 +0200 Subject: [PATCH 0041/1072] [caliper] orchestration: postprocess: factorize code --- projects/caliper/orchestration/postprocess.py | 109 +++++++----------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e6d9d8088..647ffeaba 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -59,24 +59,33 @@ def _execute_caliper_command( command: list[str], step_name: str, - script_file: Path, - log_file: Path, status_file: Path, -) -> tuple[subprocess.CompletedProcess, dict[str, Any]]: + step_logs_dir: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: """Execute a Caliper CLI command and return result and status data. Args: command: CLI command arguments list - step_name: Name for logging banners (e.g., "PARSE", "VISUALIZE") - script_file: Path to save debug script - log_file: Path to save command output + step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") status_file: Path to read status YAML from + step_logs_dir: Directory to create log file in Returns: - Tuple of (subprocess result, parsed status data) + Tuple of (subprocess result, parsed status data, log file path) """ + # Create log file path with proper step index + from projects.caliper.orchestration.step_logging import _get_next_step_index + + step_index = _get_next_step_index(step_logs_dir) + step_name_safe = step_name.replace(" ", "_") + log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Create script file for debugging + script_file = env.ARTIFACT_DIR / f"{step_name_safe}.sh" + # Log start banner and save script - log_caliper_start_banner(command, script_file, step_name) + log_caliper_start_banner(command, script_file, step_name.upper()) # Execute command with combined stdout/stderr result = subprocess.run( @@ -87,9 +96,9 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name) + status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name.upper()) - return result, status_data + return result, status_data, log_file def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: @@ -311,6 +320,7 @@ def _run_artifacts_to_kpis( plugin_module: str, base_dir: Path, manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: """Generate KPI JSON using fork/exec subprocess execution.""" @@ -331,9 +341,6 @@ def _run_artifacts_to_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "kpi_generate_status.yaml" - # Create log file - log_file = output_dir / "000__caliper_kpi_generate.log" - # Build CLI command command = build_kpi_generate_command( config=postprocess_config, @@ -344,13 +351,11 @@ def _run_artifacts_to_kpis( ) # Execute command using generic function - script_file = env.ARTIFACT_DIR / "kpi_generate.sh" - result, status_data = _execute_caliper_command( + result, status_data, log_file = _execute_caliper_command( command=command, - step_name="KPI GENERATE", - script_file=script_file, - log_file=log_file, + step_name="caliper kpi generate", status_file=status_file, + step_logs_dir=step_logs_dir, ) # Clean up temporary status file @@ -721,6 +726,7 @@ def _execute_orchestration(self) -> dict[str, Any]: else: self._run_visualize_step() logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") + self._run_kpi_and_ai_data_steps() logger.info( f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" @@ -854,7 +860,6 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - import subprocess import tempfile # Create status file for CLI output @@ -871,28 +876,12 @@ def _run_parse_step(self) -> None: use_cache=not self.config.parse.no_cache, ) - # Log start banner and save command script - script_path = env.ARTIFACT_DIR / "parse.sh" - log_caliper_start_banner(command, script_path, "parse") - - # Create log file path for stdout/stderr capture with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(self.step_logs_dir) - log_file = self.step_logs_dir / f"{step_index:03d}__caliper_parse.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Execute CLI command with output capture - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( - result, log_file, status_file, "parse" + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) if result.returncode == 0 and status_data and status_data.get("success", False): @@ -947,7 +936,6 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - import subprocess import tempfile # Create status file for CLI output @@ -977,28 +965,12 @@ def _run_visualize_step(self) -> None: use_cache=not self.config.parse.no_cache, ) - # Log start banner and save command script - script_path = env.ARTIFACT_DIR / "visualize.sh" - log_caliper_start_banner(command, script_path, "visualize") - - # Create log file path for stdout/stderr capture with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(self.step_logs_dir) - log_file = self.step_logs_dir / f"{step_index:03d}__caliper_visualize.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Execute CLI command with output capture - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( - result, log_file, status_file, "visualize" + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) if result.returncode == 0 and status_data and status_data.get("success", False): @@ -1177,7 +1149,14 @@ def _run_artifacts_to_kpis_step( if self.config.kpi.artifacts_to_kpis.enabled: with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: result = _run_artifacts_to_kpis( - self.config, plugin, model, output_dir, mod_str, self.tree_root + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, ) self._add_step("artifacts_to_kpis", result, log_file) self._check_step_result_and_set_failure("artifacts_to_kpis", result) From a514ea7db87054184b06376b4ea6018b5dd0c4a7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 11:42:03 +0200 Subject: [PATCH 0042/1072] [caliper] orchestration: launch caliper csv and ai-data as an external processes --- projects/caliper/cli/commands.py | 181 +++++++++- projects/caliper/cli/main.py | 2 + projects/caliper/cli/s3_export_cli.py | 52 ++- projects/caliper/engine/ai_eval.py | 12 +- projects/caliper/orchestration/cli_builder.py | 232 +++++++++++++ projects/caliper/orchestration/postprocess.py | 310 ++++++++++++------ 6 files changed, 671 insertions(+), 118 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index a20358eff..19dbabdd4 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -8,6 +8,7 @@ import click import yaml +from projects.caliper.cli.orchestration_entrypoints import export_kpis_to_csv_entrypoint from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export @@ -345,6 +346,10 @@ def list_reports_cmd( @click.command("ai-eval-export", hidden=True) @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option("--no-cache", is_flag=True, help="Disable parse cache") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def ai_eval_export( ctx: click.Context, @@ -352,6 +357,8 @@ def ai_eval_export( artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + no_cache: bool, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -361,12 +368,42 @@ def ai_eval_export( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: - run_ai_eval_export(base_dir=artifact_root, plugin_module=mod, plugin=plugin, output=output) + run_ai_eval_export( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=not no_cache, + ) + status_data = {"success": True, "output_file": str(output)} + click.echo(f"Exported {output}") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"ai-eval-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Exported {output}") # KPI commands @@ -432,6 +469,94 @@ def kpi_generate( sys.exit(3) +@click.command("csv-export") +@_workspace_cli_options +@click.option( + "--input", + "input_file", + type=click.Path(path_type=Path), + required=True, + help="Input KPI JSON file", +) +@click.option("--output", type=click.Path(path_type=Path), required=True, help="Output CSV file") +@click.option( + "--include-header-comments", is_flag=True, default=True, help="Include header comments in CSV" +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +@click.pass_context +def kpi_csv_export( + ctx: click.Context, + input_file: Path, + output: Path, + artifacts_dir: Path | None, + postprocess_config: Path | None, + plugin_module_override: str | None, + include_header_comments: bool, + status_file: Path | None, +) -> None: + _apply_workspace_cli_overrides( + ctx, + artifacts_dir=artifacts_dir, + postprocess_config=postprocess_config, + plugin_module_override=plugin_module_override, + ) + mod, plugin = _plugin_tuple(ctx) + + status_data = {"success": False} + + try: + # Read KPI JSON Lines file (one JSON object per line) + import json + + kpi_records = [] + with open(input_file, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: # Skip empty lines + kpi_record = json.loads(line) + kpi_records.append(kpi_record) + + # Export to CSV using the entrypoint + result_path = export_kpis_to_csv_entrypoint( + plugin=plugin, + kpi_records=kpi_records, + output_path=output, + include_header_comments=include_header_comments, + ) + + status_data = { + "success": True, + "output_file": str(result_path), + "kpi_count": len(kpi_records), + } + click.echo(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"kpi csv-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) + + @click.command("import") @click.option("--snapshot", type=click.Path(path_type=Path), required=True) @click.pass_context @@ -552,6 +677,9 @@ def kpi_analyze( "--aws-credentials-file", default="aws.credentials", help="Credentials file name within vault" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_s3_import( ctx: click.Context, @@ -565,10 +693,13 @@ def kpi_s3_import( vault: str, aws_credentials_file: str, verbose: bool, + status_file: Path | None, ) -> None: """Download historical KPI and analysis data from S3.""" from projects.core.library import vault as vault_lib + status_data = {"success": False} + try: # Ensure output directory exists output_dir.mkdir(parents=True, exist_ok=True) @@ -596,18 +727,54 @@ def kpi_s3_import( ) if result["status"] == "success": + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": result.get("files", []), + "file_count": len(result.get("files", [])), + } click.echo("✅ S3 import completed successfully") downloaded_files = result.get("files", []) click.echo(f"📄 Downloaded {len(downloaded_files)} files to {output_dir}") elif result["status"] == "empty": + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": [], + "file_count": 0, + "warning": "No objects found matching filters", + } click.echo(f"⚠️ No objects found matching filters in s3://{bucket}/{prefix}") else: - click.echo(f"❌ S3 import failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg} + click.echo(f"❌ S3 import failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) - except Exception as e: - click.echo(f"❌ S3 import failed: {e}", err=True) - sys.exit(2) + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"S3 import failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) # Artifacts commands diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 1bcb875d6..d67bc14e8 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -14,6 +14,7 @@ artifacts_export, artifacts_import, kpi_analyze, + kpi_csv_export, kpi_generate, kpi_import, kpi_s3_import, @@ -207,6 +208,7 @@ def run_cli() -> None: # Register KPI commands kpi_group.add_command(kpi_generate) +kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) kpi_group.add_command(kpi_analyze) kpi_group.add_command(kpi_s3_import) diff --git a/projects/caliper/cli/s3_export_cli.py b/projects/caliper/cli/s3_export_cli.py index 14853019f..82ca3643b 100644 --- a/projects/caliper/cli/s3_export_cli.py +++ b/projects/caliper/cli/s3_export_cli.py @@ -44,6 +44,9 @@ "--dry-run", is_flag=True, help="Show what would be uploaded without actually uploading" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def s3_export_cmd( ctx: click.Context, @@ -60,16 +63,24 @@ def s3_export_cmd( aws_credentials_file: str, dry_run: bool, verbose: bool, + status_file: Path | None, ) -> None: """Upload postprocess artifacts to S3.""" + import yaml + + status_data = {"success": False} + try: # Validate that at least one file/directory is provided if not any([kpis_file, csv_file, ai_data_dir, analysis_file]): + error_msg = "At least one file or directory must be specified" + status_data = {"success": False, "error": error_msg} click.echo("❌ Error: At least one file or directory must be specified", err=True) click.echo( " Use --kpis-file, --csv-file, --ai-data-dir, or --analysis-file", err=True ) - sys.exit(1) + if not status_file: + sys.exit(1) # Import S3 functions from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths @@ -116,7 +127,6 @@ def s3_export_cmd( ai_data_dir=ai_data_dir, analysis_file=analysis_file, bucket=bucket, - prefix=prefix, instance=instance, directory=directory, upload_id=upload_id, @@ -126,10 +136,18 @@ def s3_export_cmd( ) if result["status"] == "success": + status_data = { + "success": True, + "exported_path": result.get("exported_path", ""), + "uploaded_files": result.get("uploaded_files", 0), + "total_files": result.get("total_files", 0), + "dry_run": dry_run, + } if dry_run: click.echo("✅ Dry run completed - see upload plan") if "dry_run_file" in result: click.echo(f"📋 Upload plan saved to: {result['dry_run_file']}") + status_data["dry_run_file"] = result["dry_run_file"] else: click.echo("✅ S3 export completed successfully") if "uploaded_files" in result: @@ -138,9 +156,31 @@ def s3_export_cmd( if verbose and "exported_path" in result: click.echo(f"🌍 S3 location: {result['exported_path']}") else: - click.echo(f"❌ S3 export failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg} + click.echo(f"❌ S3 export failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) + + except Exception as e: # noqa: BLE001 + import traceback - except Exception as e: + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"❌ S3 export failed: {e}", err=True) - sys.exit(2) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + if not status_file: + sys.exit(2) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index 5a9d6953a..bafbf7447 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -27,6 +27,12 @@ def run_ai_eval_export( ) # Build base payload from plugin + if not hasattr(plugin, "build_ai_data_payload"): + raise AttributeError( + f"Plugin {plugin_module} does not support AI evaluation export. " + "Missing build_ai_data_payload method." + ) + build = plugin.build_ai_data_payload payload = build(model) @@ -75,7 +81,11 @@ def _export_test_entries_with_artifacts_engine( exported_entries = [] # Get the specific files we want to copy from the plugin - target_files = plugin.get_ai_eval_artifact_files(model) + if hasattr(plugin, "get_ai_eval_artifact_files"): + target_files = plugin.get_ai_eval_artifact_files(model) + else: + # Fallback: copy common test artifacts + target_files = ["*.log", "*.json", "*.yaml", "*.yml"] for idx, record in enumerate(model.unified_result_records): # Create directory for this test entry diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index fd5bcdea6..bfb78c716 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -184,6 +184,238 @@ def build_visualize_command( return cmd +def build_kpi_csv_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + input_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi csv-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + input_file: Input KPI JSON file + output_file: Output CSV file + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "csv-export", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # CSV export specific options + cmd.extend(["--input", str(input_file)]) + cmd.extend(["--output", str(output_file)]) + + if config.kpi.kpis_to_csv.include_header_comments: + cmd.append("--include-header-comments") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_ai_eval_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper ai-eval-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output directory for AI eval data + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "ai-eval-export", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # AI eval export specific options + cmd.extend(["--output", str(output_file)]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_import_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + output_dir: Path, +) -> list[str]: + """Build CLI command for caliper kpi s3-import. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + output_dir: Local output directory for downloads + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "s3-import", + ] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory + from projects.caliper.cli.s3_export import build_s3_prefix + + import_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if import_prefix: + cmd.extend(["--prefix", import_prefix]) + + # Create subdirectory for historical data (matches original behavior) + import_dir = output_dir / s3_config.import_.output_dir + cmd.extend(["--output-dir", str(import_dir)]) + + # Import options + if s3_config.import_.include_kpis_json: + cmd.append("--include-kpis-json") + if s3_config.import_.include_kpis_csv: + cmd.append("--include-kpis-csv") + if s3_config.import_.include_ai_data: + cmd.append("--include-ai-data") + + if s3_config.import_.max_downloads: + cmd.extend(["--max-downloads", str(s3_config.import_.max_downloads)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_export_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> list[str]: + """Build CLI command for caliper kpi s3-export. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + kpis_file: Optional KPIs JSON file + csv_file: Optional CSV file + ai_data_dir: Optional AI data directory + analysis_file: Optional analysis file + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "s3-export", + ] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory if available + from projects.caliper.cli.s3_export import build_s3_prefix + + export_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if export_prefix: + cmd.extend(["--prefix", export_prefix]) + + if s3_config.instance: + cmd.extend(["--instance", s3_config.instance]) + + if s3_config.directory: + cmd.extend(["--directory", s3_config.directory]) + + # File options + if kpis_file: + cmd.extend(["--kpis-file", str(kpis_file)]) + if csv_file: + cmd.extend(["--csv-file", str(csv_file)]) + if ai_data_dir: + cmd.extend(["--ai-data-dir", str(ai_data_dir)]) + if analysis_file: + cmd.extend(["--analysis-file", str(analysis_file)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def save_command_script(command: list[str], script_path: Path) -> None: """Save executed command to a shell script for debugging. diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 647ffeaba..ad608de18 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -21,16 +21,17 @@ from projects.caliper.cli.orchestration_entrypoints import ( analyze_kpis_entrypoint, - export_kpis_to_csv_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, - s3_export_entrypoint, - s3_import_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, + build_s3_export_command, + build_s3_import_command, build_visualize_command, handle_caliper_output_and_completion, log_caliper_start_banner, @@ -46,9 +47,6 @@ ) from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, - log_ai_data_command, - log_kpis_to_csv_command, - log_s3_export_command, step_logging, ) from projects.core.library import env @@ -96,7 +94,9 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion(result, log_file, status_file, step_name.upper()) + status_data = handle_caliper_output_and_completion( + result, log_file, status_file, step_name.upper() + ) return result, status_data, log_file @@ -368,7 +368,7 @@ def _run_artifacts_to_kpis( if status_data.get("success"): return { "status": "success", - "output_file": _make_path_relative_to_base(output_file, output_dir), + "output_file": str(output_file), "completed_at": time.time(), } else: @@ -388,7 +388,7 @@ def _run_artifacts_to_kpis( return {"status": "failed", "error": str(e), "completed_at": time.time()} -# _run_s3_import function removed - now using s3_import_entrypoint directly +# _run_s3_import function removed - now using fork/exec subprocess execution directly def _run_artifacts_to_ai_data( @@ -398,55 +398,65 @@ def _run_artifacts_to_ai_data( output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export AI evaluation payload with structured directories and copied artifacts.""" + """Export AI evaluation payload using fork/exec subprocess execution.""" try: - # Always log the step header, even for skipped operations + # Create AI data directory and output file path ai_data_dir = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + ai_data_dir.mkdir(parents=True, exist_ok=True) output_file = ai_data_dir / "ai_data_payload.json" - log_ai_data_command( - base_dir=base_dir, - plugin_module=plugin_module, + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, + use_cache=True, # Default to using cache in orchestration ) - if not hasattr(plugin, "build_ai_data_payload"): + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): return { - "status": "skipped", - "reason": "plugin does not support AI evaluation", + "status": "success", + "output_file": status_data.get("output_file", str(ai_data_dir)), + "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), } - # Create AI evaluation directory structure using configured output directory - ai_data_dir.mkdir(parents=True, exist_ok=True) - - # Build payload from plugin - payload = plugin.build_ai_data_payload(model) - - # Export structured test entries with artifact copying - exported_entries = _export_test_entries_with_artifacts(model, ai_data_dir, base_dir, plugin) - - # Add exported entries info to payload - payload["exported_test_entries"] = exported_entries - - # Write main AI eval payload - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, "w") as f: - json.dump(payload, f, indent=2) - - logger.info(f"Generated AI evaluation payload in {output_file}") - logger.info(f"Exported {len(exported_entries)} test entries with artifacts") - - return { - "status": "success", - "output_file": _make_path_relative_to_base(output_file, output_dir), - "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), - "exported_entries": len(exported_entries), - "completed_at": time.time(), - } except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} @@ -566,8 +576,11 @@ def _run_kpis_to_csv( model, output_dir: Path, kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export KPI data to CSV format using the plugin's compute_kpis method.""" + """Export KPI data to CSV format using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} @@ -579,36 +592,59 @@ def _run_kpis_to_csv( } try: - # Compute KPIs from the model - kpi_records = plugin.compute_kpis(model) - # Create output file path output_file = output_dir / postprocess_config.kpi.kpis_to_csv.output output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_kpis_to_csv_command( - input_path=kpi_json_path, - output_path=output_file, + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" + + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + input_file=kpi_json_path, + output_file=output_file, ) - # Use Caliper entrypoint for CSV export - result_path = export_kpis_to_csv_entrypoint( - plugin=plugin, - kpi_records=kpi_records, - output_path=output_file, - include_header_comments=postprocess_config.kpi.kpis_to_csv.include_header_comments, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, ) - logger.info(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") - return { - "status": "success", - "kpi_count": len(kpi_records), - "output_file": _make_path_relative_to_base(result_path, output_dir), - "completed_at": time.time(), - } + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "kpi_count": status_data.get("kpi_count", 0), + "output_file": str(output_file), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + except Exception as e: + # Log the full traceback to help with debugging + import traceback + + full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return {"status": "failed", "error": str(e), "completed_at": time.time()} @@ -1186,7 +1222,16 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: # Path to the JSON file for reference in command logging kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - result = _run_kpis_to_csv(self.config, plugin, model, output_dir, kpi_json_path) + result = _run_kpis_to_csv( + self.config, + plugin, + model, + output_dir, + kpi_json_path, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) self._add_step("kpis_to_csv", result, log_file) if result.get("status") == "failed": # CSV export failure doesn't affect overall status - it's supplementary @@ -1210,7 +1255,14 @@ def _run_artifacts_to_ai_data_step( with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: try: result = _run_artifacts_to_ai_data( - self.config, plugin, model, output_dir, mod_str, self.tree_root + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, ) self._add_step("artifacts_to_ai_data", result, log_file) @@ -1241,25 +1293,59 @@ def _run_s3_import_step(self, output_dir: Path) -> None: with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: try: - result = s3_import_entrypoint(self.config, output_dir) + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_import_status.yaml" + + # Build CLI command + command = build_s3_import_command( + config=self.config, + status_file=status_file, + output_dir=output_dir, + ) - # Standardize field names and convert paths to relative - if "import_dir" in result: - # Rename import_dir to output_dir and make relative - result["output_dir"] = _make_path_relative_to_base( - result.pop("import_dir"), output_dir - ) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 import", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("s3_import", result, log_file) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + # Get the actual import directory (where files were downloaded) + import_dir = output_dir / self.config.s3.import_.output_dir + step_result = { + "status": "success", + "output_dir": str(import_dir), + "file_count": status_data.get("file_count", 0), + "completed_at": time.time(), + } + if status_data.get("warning"): + step_result["warning"] = status_data["warning"] + else: + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + + self._add_step("s3_import", step_result, log_file) logger.info("S3 import result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info(json.dumps(step_result, indent=2, default=str)) - self._check_step_result_and_set_failure("s3_import", result) + self._check_step_result_and_set_failure("s3_import", step_result) except Exception as e: logger.exception("S3 import failed") - step_result = {"status": "failed", "error": str(e)} + step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) @@ -1394,45 +1480,61 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ): analysis_file = output_dir / analyze_step["output_file"] - # Log the CLI command to reproduce this step - s3_parent_config = self.config.s3 - export_path = f"{s3_parent_config.instance}/{s3_parent_config.directory}" + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_export_status.yaml" - log_s3_export_command( - bucket=s3_parent_config.bucket, - export_path=export_path, + # Build CLI command + command = build_s3_export_command( + config=self.config, + status_file=status_file, kpis_file=kpis_file, csv_file=csv_file, ai_data_dir=ai_data_dir, analysis_file=analysis_file, ) - result = s3_export_entrypoint( - self.config, - output_dir, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 export", + status_file=status_file, + step_logs_dir=self.step_logs_dir, ) - self._add_step("s3_export", result, log_file) - if result.get("status") == "success": + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + step_result = { + "status": "success", + "exported_path": status_data.get("exported_path", ""), + "uploaded_files": status_data.get("uploaded_files", 0), + "total_files": status_data.get("total_files", 0), + "completed_at": time.time(), + } + # Format status for better readability - duration = result.get("duration", 0) - uploaded_files = result.get("uploaded_files", 0) - failed_files = result.get("failed_files", 0) - total_files = result.get("total_files", 0) - s3_path = result.get("exported_path", "") + uploaded_files = step_result.get("uploaded_files", 0) + total_files = step_result.get("total_files", 0) + s3_path = step_result.get("exported_path", "") logger.info( - f"S3 export completed successfully in {duration}s: " - f"{uploaded_files}/{total_files} files uploaded" - f"{f' ({failed_files} failed)' if failed_files > 0 else ''} " - f"to {s3_path}" + f"S3 export completed successfully: " + f"{uploaded_files}/{total_files} files uploaded to {s3_path}" ) else: - self._check_step_result_and_set_failure("s3_export", result) + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + self._check_step_result_and_set_failure("s3_export", step_result) + + self._add_step("s3_export", step_result, log_file) except Exception as e: error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" From 3a5c54fa797c1d8a23f12fd3c53acbc7fca92fc6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 13:45:55 +0200 Subject: [PATCH 0043/1072] [caliper] orchestration: launch caliper analyse_kpis as an external process --- projects/caliper/cli/commands.py | 108 ++++++++++++++++++ projects/caliper/cli/main.py | 2 + projects/caliper/orchestration/cli_builder.py | 48 ++++++++ projects/caliper/orchestration/postprocess.py | 104 +++++++++++++---- 4 files changed, 239 insertions(+), 23 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 19dbabdd4..9cfd9ffaf 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -660,6 +660,114 @@ def kpi_analyze( sys.exit(3) +@click.command("analyse-kpis") +@_workspace_cli_options +@click.option( + "--output", + type=click.Path(path_type=Path), + required=True, + help="Output file for analysis results", +) +@click.option( + "--current-kpis-file", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Path to current KPIs JSON file", +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +@click.pass_context +def analyse_kpis_cmd( + ctx: click.Context, + output: Path, + current_kpis_file: Path, + artifacts_dir: Path | None, + postprocess_config: Path | None, + plugin_module_override: str | None, + status_file: Path | None, +) -> None: + """Analyze KPIs for orchestration (fork/exec).""" + import yaml + + _apply_workspace_cli_overrides( + ctx, + artifacts_dir=artifacts_dir, + postprocess_config=postprocess_config, + plugin_module_override=plugin_module_override, + ) + mod, plugin = _plugin_tuple(ctx) + base_dir: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + + try: + # Load manifest to get postprocess configuration + from projects.caliper.engine.plugin_config import resolve_manifest_path + from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, + ) + + manifest_path, manifest_data = resolve_manifest_path( + base_dir=base_dir, postprocess_config=_root_obj(ctx).get("postprocess_config") + ) + + # Create postprocess config object from manifest data + postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( + manifest_data.get("postprocess", {}) if manifest_data else {} + ) + + output_dir = output.parent + + # Run analysis using the entrypoint + from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint + + result = analyze_kpis_entrypoint( + postprocess_config=postprocess_config_obj, + plugin_module=mod, + base_dir=base_dir, + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + + status_data = { + "success": result.get("status") == "success", + "output_file": result.get("output_file"), + } + + if result.get("status") == "success": + click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") + else: + error_msg = result.get("error", "Unknown error") + status_data["error"] = error_msg + click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) + + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"❌ analyse-kpis failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + if not status_file: + sys.exit(2) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) + + @click.command("s3-import") @click.option("--bucket", required=True, help="S3 bucket name") @click.option("--prefix", default="", help="S3 object prefix/path") diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index d67bc14e8..0cccaeca4 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -11,6 +11,7 @@ from projects.caliper.cli.commands import ( ai_eval_export, + analyse_kpis_cmd, artifacts_export, artifacts_import, kpi_analyze, @@ -211,6 +212,7 @@ def run_cli() -> None: kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) kpi_group.add_command(kpi_analyze) +kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) # Register s3-export command under kpi group diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index bfb78c716..4018e6f95 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -346,6 +346,54 @@ def build_s3_import_command( return cmd +def build_analyse_kpis_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + current_kpis_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi analyse-kpis. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output file for analysis results + current_kpis_file: Current KPIs JSON file + + Returns: + List of command arguments for subprocess + """ + cmd = [ + sys.executable, + "-m", + "projects.caliper.cli.main", + "kpi", + "analyse-kpis", + ] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Analyse-kpis specific options + cmd.extend(["--output", str(output_file)]) + cmd.extend(["--current-kpis-file", str(current_kpis_file)]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def build_s3_export_command( config: CaliperOrchestrationPostprocessConfig, status_file: Path, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ad608de18..57339dab4 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -20,13 +20,13 @@ from pydantic import ValidationError from projects.caliper.cli.orchestration_entrypoints import ( - analyze_kpis_entrypoint, get_kpi_functions_entrypoint, load_plugin_entrypoint, parse_entrypoint, ) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, + build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -1349,6 +1349,71 @@ def _run_s3_import_step(self, output_dir: Path) -> None: self._add_step("s3_import", step_result, log_file) self._check_step_result_and_set_failure("s3_import", step_result) + +def _run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return {"status": "disabled", "reason": "analyze disabled", "completed_at": time.time()} + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": str(output_file), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + + except Exception as e: + logger.exception("KPI analysis failed in _run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time()} + def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: """Execute the KPI analysis step.""" if not self.config.analyze.enabled: @@ -1390,32 +1455,25 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: return with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: - try: - # Convert relative path to absolute for analyze function - current_kpis_path = output_dir / current_kpis_file - - result = analyze_kpis_entrypoint( - self.config, plugin_module, self.tree_root, output_dir, current_kpis_path - ) - - # Make output_file path relative - if "output_file" in result: - result["output_file"] = _make_path_relative_to_base( - result["output_file"], output_dir - ) + # current_kpis_file is now an absolute path (updated from relative path fix) + current_kpis_path = Path(current_kpis_file) - self._add_step("analyse_kpis", result, log_file) + result = _run_analyse_kpis( + postprocess_config=self.config, + output_dir=output_dir, + plugin_module=plugin_module, + base_dir=self.tree_root, + manifest_path=self.manifest_path, + current_kpis_file=current_kpis_path, + step_logs_dir=self.step_logs_dir, + ) - logger.info("KPI analysis result:") - logger.info(json.dumps(result, indent=2, default=str)) + self._add_step("analyse_kpis", result, log_file) - self._check_step_result_and_set_failure("analyse_kpis", result) + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) - except Exception as e: - logger.exception("KPI analysis failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("analyse_kpis", step_result, log_file) - self._check_step_result_and_set_failure("analyse_kpis", step_result) + self._check_step_result_and_set_failure("analyse_kpis", result) def _run_s3_export_step(self, output_dir: Path) -> None: """Execute the S3 export step.""" From e7e59188678cf2dc00455bb651daf4f7bcdf22e5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:06:11 +0200 Subject: [PATCH 0044/1072] [caliper] Improve --- projects/caliper/cli/commands.py | 153 +++-- .../caliper/cli/orchestration_entrypoints.py | 43 +- projects/caliper/cli/s3_import.py | 18 +- projects/caliper/engine/ai_eval.py | 81 ++- projects/caliper/engine/kpi/analyze.py | 25 +- projects/caliper/orchestration/cli_builder.py | 89 +-- projects/caliper/orchestration/postprocess.py | 634 +++++++++--------- .../caliper/orchestration/step_logging.py | 371 +--------- 8 files changed, 559 insertions(+), 855 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 9cfd9ffaf..583f20b29 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -147,19 +147,32 @@ def parse_cmd( show_parameter_matrix=show_matrix, ) + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + status.update( { "success": True, "plugin_module": mod, "parsed_records": len(model.unified_result_records), + "test_directories": test_directories, + "test_directories_count": len(test_directories), "cache_ref": str(model.parse_cache_ref) if model.parse_cache_ref else None, } ) click.echo( - f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" + f"Parsed {len(model.unified_result_records)} record(s) from {len(test_directories)} test directories; cache={model.parse_cache_ref}" ) + # Show test directories for user reference + if test_directories: + click.echo("📁 Test directories found:") + for test_dir in test_directories: + click.echo(f" • {test_dir}") + else: + click.echo("⚠️ No test directories found") + except Exception as e: # noqa: BLE001 status.update( { @@ -168,15 +181,15 @@ def parse_cmd( } ) click.echo(f"parse failed: {e}", err=True) - - # Write status file if requested - if status_file: - try: - status_file.parent.mkdir(parents=True, exist_ok=True) - with open(status_file, "w", encoding="utf-8") as f: - yaml.dump(status, f, default_flow_style=False, sort_keys=False) - except Exception as e: - click.echo(f"Failed to write status file {status_file}: {e}", err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) # Exit with non-zero code on failure if not status["success"]: @@ -273,15 +286,15 @@ def visualize_cmd( click.echo(f"visualize failed: {e}", err=True) click.echo("Full traceback:", err=True) click.echo(traceback.format_exc(), err=True) - - # Write status file if requested - if status_file: - try: - status_file.parent.mkdir(parents=True, exist_ok=True) - with open(status_file, "w", encoding="utf-8") as f: - yaml.dump(status, f, default_flow_style=False, sort_keys=False) - except Exception as e: - click.echo(f"Failed to write status file {status_file}: {e}", err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) # Exit with non-zero code on failure if not status["success"]: @@ -661,7 +674,6 @@ def kpi_analyze( @click.command("analyse-kpis") -@_workspace_cli_options @click.option( "--output", type=click.Path(path_type=Path), @@ -674,62 +686,70 @@ def kpi_analyze( required=True, help="Path to current KPIs JSON file", ) +@click.option( + "--historical-kpis-dir", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Directory containing historical KPI files", +) +@click.option( + "--plugin", + "plugin_module", + metavar="MODULE", + required=True, + help="Plugin module for KPI definitions and analysis rules", +) @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) -@click.pass_context def analyse_kpis_cmd( - ctx: click.Context, output: Path, current_kpis_file: Path, - artifacts_dir: Path | None, - postprocess_config: Path | None, - plugin_module_override: str | None, + historical_kpis_dir: Path, + plugin_module: str, status_file: Path | None, ) -> None: """Analyze KPIs for orchestration (fork/exec).""" import yaml - _apply_workspace_cli_overrides( - ctx, - artifacts_dir=artifacts_dir, - postprocess_config=postprocess_config, - plugin_module_override=plugin_module_override, - ) - mod, plugin = _plugin_tuple(ctx) - base_dir: Path = _root_obj(ctx)["base_dir"] - status_data = {"success": False} try: - # Load manifest to get postprocess configuration - from projects.caliper.engine.plugin_config import resolve_manifest_path + # Create a minimal postprocess config for analysis from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) - manifest_path, manifest_data = resolve_manifest_path( - base_dir=base_dir, postprocess_config=_root_obj(ctx).get("postprocess_config") - ) + # Create config with analyze enabled and using provided directories + postprocess_config_dict = { + "analyze": { + "enabled": True, + "output": output.name, # Use the provided output filename + "historical_kpis": str(historical_kpis_dir), # Use provided historical dir + } + } - # Create postprocess config object from manifest data postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( - manifest_data.get("postprocess", {}) if manifest_data else {} + postprocess_config_dict ) output_dir = output.parent - # Run analysis using the entrypoint + # Run analysis using the entrypoint (without base_dir since we have direct paths) from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint result = analyze_kpis_entrypoint( postprocess_config=postprocess_config_obj, - plugin_module=mod, - base_dir=base_dir, + plugin_module=plugin_module, + base_dir=output_dir, # Use output dir as base since we have absolute paths output_dir=output_dir, current_kpis_file=current_kpis_file, ) + # Check if the analysis function returned a proper result + if not isinstance(result, dict) or "status" not in result: + raise ValueError(f"Invalid result from analyze_kpis_entrypoint: {result}") + status_data = { "success": result.get("status") == "success", "output_file": result.get("output_file"), @@ -738,8 +758,14 @@ def analyse_kpis_cmd( if result.get("status") == "success": click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") else: - error_msg = result.get("error", "Unknown error") + # Show the full result for debugging if no clear error message + if "error" in result: + error_msg = result["error"] + else: + error_msg = f"Analysis failed with result: {result}" + status_data["error"] = error_msg + status_data.update(result) # Include all result data for debugging click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) if not status_file: sys.exit(1) @@ -834,28 +860,35 @@ def kpi_s3_import( max_downloads=max_downloads, ) + # Construct S3 path for status + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" + if result["status"] == "success": + downloaded_count = result.get("downloaded_files", 0) status_data = { "success": True, "output_dir": str(output_dir), - "downloaded_files": result.get("files", []), - "file_count": len(result.get("files", [])), + "downloaded_files": downloaded_count, + "file_count": downloaded_count, + "imported_path": s3_path, } click.echo("✅ S3 import completed successfully") - downloaded_files = result.get("files", []) - click.echo(f"📄 Downloaded {len(downloaded_files)} files to {output_dir}") - elif result["status"] == "empty": + click.echo(f"📄 Downloaded {downloaded_count} files to {output_dir}") + elif result["status"] == "warning": status_data = { "success": True, "output_dir": str(output_dir), - "downloaded_files": [], + "downloaded_files": 0, "file_count": 0, - "warning": "No objects found matching filters", + "warning": result.get("message", "No objects found matching filters"), + "imported_path": s3_path, } - click.echo(f"⚠️ No objects found matching filters in s3://{bucket}/{prefix}") + click.echo(f"⚠️ {result.get('message', 'No objects found matching filters')}") else: error_msg = result.get("error", "unknown error") - status_data = {"success": False, "error": error_msg} + status_data = {"success": False, "error": error_msg, "imported_path": s3_path} click.echo(f"❌ S3 import failed: {error_msg}", err=True) if not status_file: sys.exit(1) @@ -863,8 +896,18 @@ def kpi_s3_import( except Exception as e: # noqa: BLE001 import traceback + # Construct S3 path for error cases too + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" + full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} + status_data = { + "success": False, + "error": str(e), + "traceback": full_traceback, + "imported_path": s3_path, + } click.echo(f"S3 import failed: {e}", err=True) click.echo(f"Full traceback:\n{full_traceback}", err=True) diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py index cde8b2e86..5e319fd22 100644 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ b/projects/caliper/cli/orchestration_entrypoints.py @@ -69,15 +69,13 @@ def s3_import_entrypoint( try: from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params - from projects.caliper.orchestration.step_logging import log_s3_import_command - - # Log command to reproduce this step - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.import_ # Build import prefix using instance + directory from projects.caliper.cli.s3_export import build_s3_prefix + s3_parent_config = postprocess_config.s3 + s3_config = postprocess_config.s3.import_ + import_prefix = build_s3_prefix( instance=s3_parent_config.instance, directory=s3_parent_config.directory, @@ -85,17 +83,6 @@ def s3_import_entrypoint( ) import_dir = output_dir / s3_config.output_dir - log_s3_import_command( - bucket=s3_parent_config.bucket, - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - output_dir=import_dir, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - # Use explicit parameters instead of config dict result = run_s3_import_with_explicit_params( bucket=s3_parent_config.bucket, @@ -168,13 +155,23 @@ def analyze_kpis_entrypoint( """Entrypoint for orchestration to call KPI analysis.""" from projects.caliper.engine.kpi.analyze import analyze_kpis - return analyze_kpis( - postprocess_config=postprocess_config, - plugin_module=plugin_module, - base_dir=base_dir, - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) + try: + result = analyze_kpis( + postprocess_config=postprocess_config, + plugin_module=plugin_module, + base_dir=base_dir, + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + return result + except Exception as e: + # Don't suppress the exception - let it bubble up with full traceback + import sys + import traceback + + print(f"❌ Exception in analyze_kpis_entrypoint: {e}", file=sys.stderr) + print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) + raise def kpi_generate_entrypoint( diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index db6349fd8..e5d3bc9c9 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -25,14 +25,13 @@ logger = logging.getLogger(__name__) -def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000) -> list[dict]: +def list_s3_objects(s3_client, bucket: str, prefix: str) -> list[dict]: """List objects in S3 bucket with given prefix. Args: s3_client: Configured boto3 S3 client bucket: S3 bucket name prefix: S3 key prefix to filter objects - max_objects: Maximum number of objects to return Returns: List of S3 object metadata dictionaries @@ -41,9 +40,8 @@ def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000 objects = [] paginator = s3_client.get_paginator("list_objects_v2") - for page in paginator.paginate( - Bucket=bucket, Prefix=prefix, PaginationConfig={"MaxItems": max_objects} - ): + # List ALL objects first - don't limit listing, limit downloads after filtering + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): if "Contents" in page: for obj in page["Contents"]: objects.append( @@ -273,7 +271,7 @@ def run_s3_import_with_explicit_params( } # List S3 objects (even in dry run mode to show what would be downloaded) - objects = list_s3_objects(s3_client, bucket, import_s3_prefix, max_downloads) + objects = list_s3_objects(s3_client, bucket, import_s3_prefix) logger.info(f"Found {len(objects)} total objects in s3://{bucket}/{import_s3_prefix}") @@ -293,6 +291,14 @@ def __init__(self): download_objects = filter_objects_for_download(objects, FilterConfig()) logger.info(f"After filtering: {len(download_objects)} objects match the import filters") + # Sort by last_modified timestamp in descending order (most recent first) + download_objects.sort(key=lambda obj: obj["last_modified"], reverse=True) + + # Apply max_downloads limit after filtering to get the most relevant files + if len(download_objects) > max_downloads: + logger.info(f"Limiting downloads to first {max_downloads} matching files") + download_objects = download_objects[:max_downloads] + if not download_objects: logger.warning("No objects match the import filters") return { diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index bafbf7447..f22cea9ad 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -103,29 +103,68 @@ def _export_test_entries_with_artifacts_engine( # Copy target files if they exist for target_file in target_files: - source_file = base_dir / target_file - if source_file.exists(): - # Create target directory structure in test entry dir - target_path = test_entry_dir / Path(target_file).name - target_path.parent.mkdir(parents=True, exist_ok=True) - - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "size_bytes": source_file.stat().st_size, - } + # Handle plugin-provided paths directly, expand fallback patterns + if hasattr(plugin, "get_ai_eval_artifact_files"): + # Plugin provided specific files - treat as direct paths + source_file = base_dir / target_file + if source_file.exists(): + # Create target directory structure in test entry dir + target_path = test_entry_dir / Path(target_file).name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(source_file, target_path) + entry_info["copied_files"].append( + { + "source": str(source_file), + "target": str(target_path), + "size_bytes": source_file.stat().st_size, + } + ) + logger.debug(f"Copied {source_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {source_file}: {e}") + entry_info["missing_files"].append( + {"file": str(source_file), "error": str(e)} + ) + else: + entry_info["missing_files"].append( + {"file": str(source_file), "error": "File does not exist"} ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) + # Fallback mode - expand glob patterns + import glob + + pattern_path = base_dir / target_file + matched_files = glob.glob(str(pattern_path)) + + if matched_files: + # Copy all matched files + for matched_file_str in matched_files: + matched_file = Path(matched_file_str) + target_path = test_entry_dir / matched_file.name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(matched_file, target_path) + entry_info["copied_files"].append( + { + "source": str(matched_file), + "target": str(target_path), + "size_bytes": matched_file.stat().st_size, + } + ) + logger.debug(f"Copied {matched_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {matched_file}: {e}") + entry_info["missing_files"].append( + {"file": str(matched_file), "error": str(e)} + ) + else: + # Pattern matched no files + entry_info["missing_files"].append( + {"file": str(pattern_path), "error": "Pattern matched no files"} + ) # Write entry metadata entry_metadata_file = test_entry_dir / "entry_metadata.json" diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 8b96ab459..40ae18680 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -156,16 +156,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Log command to reproduce this step - from projects.caliper.orchestration.step_logging import log_analyze_command - - log_analyze_command( - base_dir=base_dir, - plugin_module=plugin_module, - current_kpis_path=current_kpis_file, - historical_kpis_dir=historical_kpis_dir, - output_path=output_path, - ) # Call the core analysis function exit_code = run_kpi_analysis( @@ -237,12 +227,15 @@ def analyze_kpis( except Exception as e: error_msg = f"{type(e).__name__}: {str(e)}" logger.exception(f"Analysis step failed with {error_msg}") - return { - "status": "failed", - "error": error_msg, - "error_type": type(e).__name__, - "completed_at": time.time(), - } + + # Don't suppress the exception - provide detailed error info and re-raise + import traceback + + full_traceback = traceback.format_exc() + logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") + + # Re-raise the original exception so the caller can see the full details + raise RuntimeError(f"KPI analysis failed: {error_msg}") from e def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 4018e6f95..7e12fd4f4 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -7,7 +7,6 @@ import logging import subprocess -import sys from pathlib import Path from projects.caliper.orchestration.postprocess_config import ( @@ -16,6 +15,9 @@ logger = logging.getLogger(__name__) +# Base command for all Caliper CLI invocations +_CALIPER_BASE_CMD = ["caliper"] + def build_parse_command( config: CaliperOrchestrationPostprocessConfig, @@ -36,12 +38,7 @@ def build_parse_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "parse", - ] + cmd = _CALIPER_BASE_CMD + ["parse"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -84,13 +81,7 @@ def build_kpi_generate_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "generate", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "generate"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -131,12 +122,7 @@ def build_visualize_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "visualize", - ] + cmd = _CALIPER_BASE_CMD + ["visualize"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -151,7 +137,7 @@ def build_visualize_command( cmd.extend(["--output-dir", str(output_dir)]) if config.visualize.reports: - cmd.extend(["--reports", ",".join(config.visualize.reports)]) + cmd.extend(["--reports", config.visualize.reports]) if config.visualize.report_group: cmd.extend(["--report-group", config.visualize.report_group]) @@ -205,13 +191,7 @@ def build_kpi_csv_export_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "csv-export", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "csv-export"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -255,12 +235,7 @@ def build_ai_eval_export_command( Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "ai-eval-export", - ] + cmd = _CALIPER_BASE_CMD + ["ai-eval-export"] # Workspace options cmd.extend(["--artifacts-dir", str(tree_root)]) @@ -300,13 +275,7 @@ def build_s3_import_command( """ s3_config = config.s3 - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "s3-import", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-import"] # S3 configuration cmd.extend(["--bucket", s3_config.bucket]) @@ -353,40 +322,34 @@ def build_analyse_kpis_command( status_file: Path, output_file: Path, current_kpis_file: Path, + historical_kpis_dir: Path, ) -> list[str]: """Build CLI command for caliper kpi analyse-kpis. Args: config: Orchestration configuration - tree_root: Base directory for artifacts - manifest_path: Optional manifest file path + tree_root: Base directory for artifacts (unused, kept for compatibility) + manifest_path: Optional manifest file path (unused, kept for compatibility) status_file: Where to write status YAML output_file: Output file for analysis results current_kpis_file: Current KPIs JSON file + historical_kpis_dir: Directory containing historical KPI files Returns: List of command arguments for subprocess """ - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "analyse-kpis", - ] - - # Workspace options - cmd.extend(["--artifacts-dir", str(tree_root)]) - - if manifest_path: - cmd.extend(["--postprocess-config", str(manifest_path)]) - - if config.plugin_module: - cmd.extend(["--plugin", config.plugin_module]) + cmd = _CALIPER_BASE_CMD + ["kpi", "analyse-kpis"] # Analyse-kpis specific options cmd.extend(["--output", str(output_file)]) cmd.extend(["--current-kpis-file", str(current_kpis_file)]) + cmd.extend(["--historical-kpis-dir", str(historical_kpis_dir)]) + + # Plugin module is required + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + else: + raise ValueError("Plugin module is required for KPI analysis") # Status file for orchestration cmd.extend(["--status-file", str(status_file)]) @@ -417,13 +380,7 @@ def build_s3_export_command( """ s3_config = config.s3 - cmd = [ - sys.executable, - "-m", - "projects.caliper.cli.main", - "kpi", - "s3-export", - ] + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-export"] # S3 configuration cmd.extend(["--bucket", s3_config.bucket]) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 57339dab4..024fb5e89 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -47,7 +47,6 @@ ) from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, - step_logging, ) from projects.core.library import env @@ -325,12 +324,18 @@ def _run_artifacts_to_kpis( """Generate KPI JSON using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } if not postprocess_config.kpi.artifacts_to_kpis.enabled: return { "status": "disabled", "reason": "kpi.artifacts_to_kpis disabled", "completed_at": time.time(), + "log_file": None, } try: @@ -370,12 +375,14 @@ def _run_artifacts_to_kpis( "status": "success", "output_file": str(output_file), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -385,7 +392,7 @@ def _run_artifacts_to_kpis( full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} # _run_s3_import function removed - now using fork/exec subprocess execution directly @@ -442,12 +449,14 @@ def _run_artifacts_to_ai_data( "output_file": status_data.get("output_file", str(ai_data_dir)), "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -457,7 +466,7 @@ def _run_artifacts_to_ai_data( full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} def _load_test_labels(test_dir: Path) -> dict[str, Any]: @@ -583,12 +592,18 @@ def _run_kpis_to_csv( """Export KPI data to CSV format using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } if not postprocess_config.kpi.kpis_to_csv.enabled: return { "status": "disabled", "reason": "kpi.kpis_to_csv disabled", "completed_at": time.time(), + "log_file": None, } try: @@ -630,12 +645,14 @@ def _run_kpis_to_csv( "kpi_count": status_data.get("kpi_count", 0), "output_file": str(output_file), "completed_at": time.time(), + "log_file": log_file, } else: return { "status": "failed", "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), + "log_file": log_file, } except Exception as e: @@ -645,7 +662,85 @@ def _run_kpis_to_csv( full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def _run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return { + "status": "disabled", + "reason": "analyze disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": str(output_file), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("KPI analysis failed in _run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} class CaliperPostprocessOrchestrator: @@ -927,6 +1022,8 @@ def _run_parse_step(self) -> None: "status": "success", "plugin_module": status_data.get("plugin_module", "unknown"), "record_count": status_data.get("parsed_records", 0), + "test_directories": status_data.get("test_directories", []), + "test_directories_count": status_data.get("test_directories_count", 0), "parse_cache_ref": status_data.get("cache_ref"), "completed_at": time.time(), }, @@ -1183,19 +1280,19 @@ def _run_artifacts_to_kpis_step( ) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: - with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir) as log_file: - result = _run_artifacts_to_kpis( - self.config, - plugin, - model, - output_dir, - mod_str, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("artifacts_to_kpis", result, log_file) - self._check_step_result_and_set_failure("artifacts_to_kpis", result) + result = _run_artifacts_to_kpis( + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_kpis", result, log_file) + self._check_step_result_and_set_failure("artifacts_to_kpis", result) else: self._add_step( "artifacts_to_kpis", @@ -1219,23 +1316,23 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - with step_logging("caliper_kpis_to_csv", self.step_logs_dir) as log_file: - # Path to the JSON file for reference in command logging - kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - result = _run_kpis_to_csv( - self.config, - plugin, - model, - output_dir, - kpi_json_path, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("kpis_to_csv", result, log_file) - if result.get("status") == "failed": - # CSV export failure doesn't affect overall status - it's supplementary - logger.warning("KPI CSV export failed but continuing execution") + # Path to the JSON file for reference in command logging + kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + result = _run_kpis_to_csv( + self.config, + plugin, + model, + output_dir, + kpi_json_path, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("kpis_to_csv", result, log_file) + if result.get("status") == "failed": + # CSV export failure doesn't affect overall status - it's supplementary + logger.warning("KPI CSV export failed but continuing execution") def _run_artifacts_to_ai_data_step( self, plugin: Any, model: Any, output_dir: Path, mod_str: str @@ -1252,31 +1349,31 @@ def _run_artifacts_to_ai_data_step( ) return - with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir) as log_file: - try: - result = _run_artifacts_to_ai_data( - self.config, - plugin, - model, - output_dir, - mod_str, - self.tree_root, - self.manifest_path, - self.step_logs_dir, - ) - self._add_step("artifacts_to_ai_data", result, log_file) + try: + result = _run_artifacts_to_ai_data( + self.config, + plugin, + model, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_ai_data", result, log_file) - logger.info("AI eval export result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info("AI eval export result:") + logger.info(json.dumps(result, indent=2, default=str)) - # Check if the result indicates failure or warning - self._check_step_result_and_set_failure("artifacts_to_ai_data", result) + # Check if the result indicates failure or warning + self._check_step_result_and_set_failure("artifacts_to_ai_data", result) - except Exception as e: - logger.exception("AI eval export failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("artifacts_to_ai_data", step_result, log_file) - self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) + except Exception as e: + logger.exception("AI eval export failed") + step_result = {"status": "failed", "error": str(e)} + self._add_step("artifacts_to_ai_data", step_result, None) + self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) def _run_s3_import_step(self, output_dir: Path) -> None: """Execute the S3 import step.""" @@ -1291,128 +1388,66 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_import", self.step_logs_dir) as log_file: - try: - # Create temporary status file for subprocess communication - status_file = output_dir / "s3_import_status.yaml" - - # Build CLI command - command = build_s3_import_command( - config=self.config, - status_file=status_file, - output_dir=output_dir, - ) - - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper s3 import", - status_file=status_file, - step_logs_dir=self.step_logs_dir, - ) - - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass - - # Convert to expected format - if status_data.get("success"): - # Get the actual import directory (where files were downloaded) - import_dir = output_dir / self.config.s3.import_.output_dir - step_result = { - "status": "success", - "output_dir": str(import_dir), - "file_count": status_data.get("file_count", 0), - "completed_at": time.time(), - } - if status_data.get("warning"): - step_result["warning"] = status_data["warning"] - else: - step_result = { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } - - self._add_step("s3_import", step_result, log_file) - - logger.info("S3 import result:") - logger.info(json.dumps(step_result, indent=2, default=str)) - - self._check_step_result_and_set_failure("s3_import", step_result) - - except Exception as e: - logger.exception("S3 import failed") - step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} - self._add_step("s3_import", step_result, log_file) - self._check_step_result_and_set_failure("s3_import", step_result) - - -def _run_analyse_kpis( - postprocess_config: CaliperOrchestrationPostprocessConfig, - output_dir: Path, - plugin_module: str, - base_dir: Path, - manifest_path: Path | None, - current_kpis_file: Path, - step_logs_dir: Path, -) -> dict[str, Any]: - """Analyze KPIs using fork/exec subprocess execution.""" + try: + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_import_status.yaml" - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled", "completed_at": time.time()} + # Build CLI command + command = build_s3_import_command( + config=self.config, + status_file=status_file, + output_dir=output_dir, + ) - try: - # Prepare paths - output_file = output_dir / postprocess_config.analyze.output - output_file.parent.mkdir(parents=True, exist_ok=True) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 import", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - # Create temporary status file for subprocess communication - status_file = output_dir / "analyse_kpis_status.yaml" + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - # Build CLI command - command = build_analyse_kpis_command( - config=postprocess_config, - tree_root=base_dir, - manifest_path=manifest_path, - status_file=status_file, - output_file=output_file, - current_kpis_file=current_kpis_file, - ) + # Convert to expected format + if status_data.get("success"): + # Get the actual import directory (where files were downloaded) + import_dir = output_dir / self.config.s3.import_.output_dir + step_result = { + "status": "success", + "output_dir": str(import_dir), + "file_count": status_data.get("file_count", 0), + "completed_at": time.time(), + } + if status_data.get("warning"): + step_result["warning"] = status_data["warning"] + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] + else: + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper kpi analyse-kpis", - status_file=status_file, - step_logs_dir=step_logs_dir, - ) + self._add_step("s3_import", step_result, log_file) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass + logger.info("S3 import result:") + logger.info(json.dumps(step_result, indent=2, default=str)) - # Convert to expected format - if status_data.get("success"): - return { - "status": "success", - "output_file": str(output_file), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } + self._check_step_result_and_set_failure("s3_import", step_result) - except Exception as e: - logger.exception("KPI analysis failed in _run_analyse_kpis") - return {"status": "failed", "error": str(e), "completed_at": time.time()} + except Exception as e: + logger.exception("S3 import failed") + step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} + self._add_step("s3_import", step_result, None) + self._check_step_result_and_set_failure("s3_import", step_result) def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: """Execute the KPI analysis step.""" @@ -1454,26 +1489,26 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - with step_logging("caliper_analyse_kpis", self.step_logs_dir) as log_file: - # current_kpis_file is now an absolute path (updated from relative path fix) - current_kpis_path = Path(current_kpis_file) - - result = _run_analyse_kpis( - postprocess_config=self.config, - output_dir=output_dir, - plugin_module=plugin_module, - base_dir=self.tree_root, - manifest_path=self.manifest_path, - current_kpis_file=current_kpis_path, - step_logs_dir=self.step_logs_dir, - ) + # current_kpis_file is now an absolute path (updated from relative path fix) + current_kpis_path = Path(current_kpis_file) + + result = _run_analyse_kpis( + postprocess_config=self.config, + output_dir=output_dir, + plugin_module=plugin_module, + base_dir=self.tree_root, + manifest_path=self.manifest_path, + current_kpis_file=current_kpis_path, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("analyse_kpis", result, log_file) + log_file = result.pop("log_file", None) + self._add_step("analyse_kpis", result, log_file) - logger.info("KPI analysis result:") - logger.info(json.dumps(result, indent=2, default=str)) + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) - self._check_step_result_and_set_failure("analyse_kpis", result) + self._check_step_result_and_set_failure("analyse_kpis", result) def _run_s3_export_step(self, output_dir: Path) -> None: """Execute the S3 export step.""" @@ -1488,132 +1523,131 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_export", self.step_logs_dir) as log_file: - try: - # Collect file paths from previous steps - kpis_file = None - csv_file = None - ai_data_dir = None - analysis_file = None - - # Get KPI JSON file from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if ( - artifacts_to_kpis_step - and artifacts_to_kpis_step.get("status") == "success" - and artifacts_to_kpis_step.get("output_file") - ): - kpis_file = output_dir / artifacts_to_kpis_step["output_file"] - - # Get CSV file from kpis_to_csv step - kpis_to_csv_step = self._get_step("kpis_to_csv") - if ( - kpis_to_csv_step - and kpis_to_csv_step.get("status") == "success" - and kpis_to_csv_step.get("output_file") - ): - csv_file = output_dir / kpis_to_csv_step["output_file"] - - # Get AI data directory from artifacts_to_ai_data step - ai_data_step = self._get_step("artifacts_to_ai_data") - logger.info(f"AI data step found: {ai_data_step}") - if ( - ai_data_step - and ai_data_step.get("status") == "success" - and ai_data_step.get("ai_data_dir") - ): - ai_data_dir = output_dir / ai_data_step["ai_data_dir"] - logger.info(f"AI data directory set to: {ai_data_dir}") - else: - logger.warning( - f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" - ) - - # Get analysis file from analyse_kpis step - analyze_step = self._get_step("analyse_kpis") - if ( - analyze_step - and analyze_step.get("status") in ("success", "warning") - and analyze_step.get("output_file") - ): - analysis_file = output_dir / analyze_step["output_file"] - - # Create temporary status file for subprocess communication - status_file = output_dir / "s3_export_status.yaml" - - # Build CLI command - command = build_s3_export_command( - config=self.config, - status_file=status_file, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, - ) - - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper s3 export", - status_file=status_file, - step_logs_dir=self.step_logs_dir, + try: + # Collect file paths from previous steps + kpis_file = None + csv_file = None + ai_data_dir = None + analysis_file = None + + # Get KPI JSON file from artifacts_to_kpis step + artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") + if ( + artifacts_to_kpis_step + and artifacts_to_kpis_step.get("status") == "success" + and artifacts_to_kpis_step.get("output_file") + ): + kpis_file = output_dir / artifacts_to_kpis_step["output_file"] + + # Get CSV file from kpis_to_csv step + kpis_to_csv_step = self._get_step("kpis_to_csv") + if ( + kpis_to_csv_step + and kpis_to_csv_step.get("status") == "success" + and kpis_to_csv_step.get("output_file") + ): + csv_file = output_dir / kpis_to_csv_step["output_file"] + + # Get AI data directory from artifacts_to_ai_data step + ai_data_step = self._get_step("artifacts_to_ai_data") + logger.info(f"AI data step found: {ai_data_step}") + if ( + ai_data_step + and ai_data_step.get("status") == "success" + and ai_data_step.get("ai_data_dir") + ): + ai_data_dir = output_dir / ai_data_step["ai_data_dir"] + logger.info(f"AI data directory set to: {ai_data_dir}") + else: + logger.warning( + f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass + # Get analysis file from analyse_kpis step + analyze_step = self._get_step("analyse_kpis") + if ( + analyze_step + and analyze_step.get("status") in ("success", "warning") + and analyze_step.get("output_file") + ): + analysis_file = output_dir / analyze_step["output_file"] - # Convert to expected format - if status_data.get("success"): - step_result = { - "status": "success", - "exported_path": status_data.get("exported_path", ""), - "uploaded_files": status_data.get("uploaded_files", 0), - "total_files": status_data.get("total_files", 0), - "completed_at": time.time(), - } + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_export_status.yaml" - # Format status for better readability - uploaded_files = step_result.get("uploaded_files", 0) - total_files = step_result.get("total_files", 0) - s3_path = step_result.get("exported_path", "") + # Build CLI command + command = build_s3_export_command( + config=self.config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) - logger.info( - f"S3 export completed successfully: " - f"{uploaded_files}/{total_files} files uploaded to {s3_path}" - ) - else: - step_result = { - "status": "failed", - "error": status_data.get("error", "Unknown error"), - "completed_at": time.time(), - } - self._check_step_result_and_set_failure("s3_export", step_result) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 export", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("s3_export", step_result, log_file) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - except Exception as e: - error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" + # Convert to expected format + if status_data.get("success"): + step_result = { + "status": "success", + "exported_path": status_data.get("exported_path", ""), + "uploaded_files": status_data.get("uploaded_files", 0), + "total_files": status_data.get("total_files", 0), + "completed_at": time.time(), + } - # Log programming errors more prominently - if isinstance(e, AttributeError | NameError | TypeError): - logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") - logger.exception("Full traceback for programming error:") - else: - logger.error(error_msg) - logger.exception("Full traceback:") + # Format status for better readability + uploaded_files = step_result.get("uploaded_files", 0) + total_files = step_result.get("total_files", 0) + s3_path = step_result.get("exported_path", "") + logger.info( + f"S3 export completed successfully: " + f"{uploaded_files}/{total_files} files uploaded to {s3_path}" + ) + else: step_result = { "status": "failed", - "error": error_msg, - "exception_type": type(e).__name__, + "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), } - self._add_step("s3_export", step_result, log_file) self._check_step_result_and_set_failure("s3_export", step_result) + self._add_step("s3_export", step_result, log_file) + + except Exception as e: + error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" + + # Log programming errors more prominently + if isinstance(e, AttributeError | NameError | TypeError): + logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") + logger.exception("Full traceback for programming error:") + else: + logger.error(error_msg) + logger.exception("Full traceback:") + + step_result = { + "status": "failed", + "error": error_msg, + "exception_type": type(e).__name__, + "completed_at": time.time(), + } + self._add_step("s3_export", step_result, None) + self._check_step_result_and_set_failure("s3_export", step_result) + def _compute_final_status(self) -> str: """Compute the final postprocessing status.""" # Debug logging to identify what's causing failures diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 7ef0f6d17..5354c48e0 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -237,372 +237,7 @@ def cleanup_step_logging(): del _step_local_handlers.file_handler -def _log_command_banner( - step_name: str, - command_line: str, - description: str = "", - step_args: dict[str, Any] | None = None, -) -> None: - """ - Log a command reproduction banner at the start of a step. - Args: - step_name: Name of the step (e.g., "caliper_parse") - command_line: The CLI command to reproduce this step - description: Brief description of what this step does - step_args: Dictionary of step arguments for reference - """ - # Use root logger to ensure step handler captures the banner - logger = logging.getLogger() - - LINE_WIDTH = 80 - - logger.info("") - logger.info("=" * LINE_WIDTH) - logger.info(f"CALIPER STEP: {step_name.upper()}") - if description: - logger.info(f"DESCRIPTION: {description}") - logger.info("=" * LINE_WIDTH) - logger.info(f"COMMAND TO REPRODUCE:\n{command_line}") - - if step_args: - logger.info("") - logger.info("STEP PARAMETERS:") - for key, value in step_args.items(): - if value is not None: - # Truncate very long values - value_str = str(value) - if len(value_str) > 100: - value_str = value_str[:97] + "..." - logger.info(f" {key}: {value_str}") - - logger.info("=" * LINE_WIDTH) - logger.info("") - - -def log_parse_command( - base_dir: Path, - plugin_module: str, - use_cache: bool, - manifest_path: Path | None = None, -) -> None: - """Log the CLI command to reproduce the parse step.""" - cache_flag = "" if use_cache else " --no-cache" - command = f'caliper parse --base-dir "{base_dir}" --plugin {plugin_module}{cache_flag}' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "use_cache": use_cache, - } - - # Only include manifest_path if it's provided and safe to log - if manifest_path: - step_args["manifest_path"] = str(manifest_path) - - _log_command_banner( - "caliper_parse", - command, - "Parse test results from artifact tree into unified model", - step_args, - ) - - -def log_visualize_command( - base_dir: Path, - plugin_module: str, - output_dir: Path, - reports_csv: str | None = None, - report_group: str | None = None, - visualize_config_path: Path | None = None, - include_pairs: tuple[str, ...] = (), - exclude_pairs: tuple[str, ...] = (), - use_cache: bool = True, -) -> None: - """Log the CLI command to reproduce the visualize step.""" - command_parts = [ - "caliper visualize", - f'--base-dir "{base_dir}"', - f'--output-dir "{output_dir}"', - f"--plugin {plugin_module}", - ] - - if not use_cache: - command_parts.append("--no-cache") - if reports_csv: - command_parts.append(f'--reports "{reports_csv}"') - if report_group: - command_parts.append(f'--report-group "{report_group}"') - if visualize_config_path: - command_parts.append(f'--visualize-config "{visualize_config_path}"') - - for include_pair in include_pairs: - command_parts.append(f'--include "{include_pair}"') - for exclude_pair in exclude_pairs: - command_parts.append(f'--exclude "{exclude_pair}"') - - command = " \\\n ".join(command_parts) - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_dir": str(output_dir), - "use_cache": use_cache, - } - - # Only include optional parameters if they're provided - if reports_csv: - step_args["reports_csv"] = reports_csv - if report_group: - step_args["report_group"] = report_group - if visualize_config_path: - step_args["visualize_config_path"] = str(visualize_config_path) - if include_pairs: - step_args["include_pairs"] = list(include_pairs) - if exclude_pairs: - step_args["exclude_pairs"] = list(exclude_pairs) - - _log_command_banner( - "caliper_visualize", - command, - "Generate visualization reports and plots from parsed data", - step_args, - ) - - -def log_artifacts_to_kpis_command( - base_dir: Path, - plugin_module: str, - output_file: Path, -) -> None: - """Log the CLI command to reproduce the KPI generation step.""" - command = f'caliper kpi generate --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_file": str(output_file), - } - - _log_command_banner( - "caliper_kpi_generate", command, "Generate KPI metrics from parsed test results", step_args - ) - - -def log_kpi_export_command( - input_path: Path, - target_system: str = "opensearch", -) -> None: - """Log the CLI command to reproduce the KPI export step.""" - command = f'caliper kpi export --input "{input_path}" --target {target_system}' - - step_args = { - "input_path": str(input_path), - "target_system": target_system, - } - - _log_command_banner( - "caliper_kpi_export", command, "Export KPI metrics to external monitoring system", step_args - ) - - -def log_ai_data_command( - base_dir: Path, - plugin_module: str, - output_file: Path, -) -> None: - """Log the CLI command to reproduce the AI evaluation export step.""" - command = f'caliper ai-eval export --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' - - step_args = { - "base_dir": str(base_dir), - "plugin_module": plugin_module, - "output_file": str(output_file), - } - - _log_command_banner( - "caliper_ai_data_export", - command, - "Export AI evaluation payload with structured test directories and artifacts", - step_args, - ) - - -def log_s3_import_command( - bucket: str, - instance: str | None, - directory: str | None, - output_dir: Path, - include_kpis_json: bool = False, - include_kpis_csv: bool = False, - include_ai_data: bool = False, - max_downloads: int = 50, -) -> None: - """Log the CLI command to reproduce the S3 import step.""" - # Build prefix from instance and directory - from projects.caliper.cli.s3_export import build_s3_prefix - - prefix = build_s3_prefix(instance=instance, directory=directory, trailing_slash=False) - - command_parts = [ - f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' - ] - - # Add include flags if they are True (since default is False) - if include_kpis_json: - command_parts.append("--include-kpis-json") - if include_kpis_csv: - command_parts.append("--include-kpis-csv") - if include_ai_data: - command_parts.append("--include-ai-data") - if max_downloads != 50: - command_parts.append(f"--max-downloads {max_downloads}") - - command = " ".join(command_parts) - - step_args = { - "bucket": bucket, - "instance": instance, - "directory": directory, - "prefix": prefix, - "output_dir": str(output_dir), - "include_kpis_json": include_kpis_json, - "include_kpis_csv": include_kpis_csv, - "include_ai_data": include_ai_data, - "max_downloads": max_downloads, - } - - _log_command_banner( - "caliper_s3_import", - command, - "Import historical data from S3 for analysis", - step_args, - ) - - -def log_s3_export_command( - bucket: str, - export_path: str, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, -) -> None: - """Log the CLI command to reproduce the S3 export step.""" - command_parts = [f'caliper s3-export --bucket "{bucket}"'] - - # Add file paths - if kpis_file: - command_parts.append(f'--kpis-file "{kpis_file}"') - if csv_file: - command_parts.append(f'--csv-file "{csv_file}"') - if ai_data_dir: - command_parts.append(f'--ai-data-dir "{ai_data_dir}"') - if analysis_file: - command_parts.append(f'--analysis-file "{analysis_file}"') - - # Parse export_path once and use for both command and step_args - instance = None - directory = None - prefix = None - - if export_path: - # Parse the export path to get prefix, instance, directory - parts = export_path.split("/") - if len(parts) >= 3: - instance = parts[0] - directory = parts[1] - prefix = "/".join(parts[2:]) if len(parts) > 2 else "" - command_parts.append(f'--instance "{instance}"') - command_parts.append(f'--directory "{directory}"') - if prefix: - command_parts.append(f'--prefix "{prefix}"') - - command = " ".join(command_parts) - - step_args = { - "bucket": bucket, - "kpis_file": str(kpis_file) if kpis_file else None, - "csv_file": str(csv_file) if csv_file else None, - "ai_data_dir": str(ai_data_dir) if ai_data_dir else None, - "analysis_file": str(analysis_file) if analysis_file else None, - } - - # Add command line parameters to step_args (same values used in command) - if instance: - step_args["instance"] = instance - if directory: - step_args["directory"] = directory - if prefix: - step_args["prefix"] = prefix - - _log_command_banner( - "caliper_s3_export", - command, - "Export postprocess artifacts to S3", - step_args, - ) - - -def log_analyze_command( - base_dir: Path, - plugin_module: str, - current_kpis_path: Path | None = None, - historical_kpis_dir: Path | None = None, - output_path: Path | None = None, -) -> None: - """Log the CLI command to reproduce the analyze step.""" - - # Use the correct command structure: caliper kpi analyze - command = "caliper kpi analyze" - - # Add current KPI file path if provided - if current_kpis_path: - command += f' --current "{current_kpis_path}"' - - # Add the historical directory, not the specific file - if historical_kpis_dir: - command += f' --baseline-dir "{historical_kpis_dir}"' - - # Add output path if provided - if output_path: - command += f' --output "{output_path}"' - - # Add plugin module - command += f" --plugin {plugin_module}" - - # Add helpful context - command += "\n# Analyzes KPI for regressions" - command += "\n# The tool will automatically find the most recent kpis.json file in the baseline directory" - command += "\n# To re-run this analysis step independently, use the command above" - command += f"\n# Plugin: {plugin_module}" - command += f"\n# Base dir: {base_dir}" - - step_args = { - "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, - "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, - "output_path": str(output_path) if output_path else None, - "plugin_module": plugin_module, - "base_dir": str(base_dir), - } - - _log_command_banner( - "caliper_analyze", command, "Run regression analysis on test results", step_args - ) - - -def log_kpis_to_csv_command( - input_path: Path, - output_path: Path, -) -> None: - """Log the CLI command to reproduce the KPI CSV export step.""" - command = f'caliper kpi csv-export --input "{input_path}" --output "{output_path}"' - - step_args = { - "input_path": str(input_path), - "output_path": str(output_path), - } - - _log_command_banner( - "caliper_kpi_csv_export", command, "Export KPI metrics to CSV format", step_args - ) + + + From a55a40a7b261e053018bacdf910dfc9d57ab7c60 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:43:06 +0200 Subject: [PATCH 0045/1072] fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs --- .../base/workflows/task-forge-step.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index dcb29b30c..72e58e9c0 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,6 +13,8 @@ spec: type: string - name: kubeconfig-secret type: string + default: "fournos-clusterless-placeholder" + description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -31,6 +33,8 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig + - name: KUBECONFIG_SECRET_PARAM + value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -73,7 +77,24 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - export KUBECONFIG=$TARGET_KUBECONFIG + # Check if we have a kubeconfig for target cluster + if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then + echo "Setting up target cluster kubeconfig..." + export KUBECONFIG=$TARGET_KUBECONFIG + export CLUSTERLESS_MODE="false" + echo "Target cluster info:" + kubectl cluster-info + else + if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then + echo "Running in clusterless mode - no target cluster kubeconfig" + else + echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" + echo "Running in clusterless mode as fallback" + fi + export CLUSTERLESS_MODE="true" + echo "Hub cluster info:" + kubectl cluster-info + fi # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -126,3 +147,4 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) + optional: true From 9402f47336e9b5a0a9cd3e879249f285bfa6b7ad Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:49:17 +0200 Subject: [PATCH 0046/1072] [caliper] orchestration: postprocess: save the step scripts in a dedicated directory --- projects/caliper/orchestration/postprocess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 024fb5e89..4f0645274 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -78,8 +78,10 @@ def _execute_caliper_command( log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" log_file.parent.mkdir(parents=True, exist_ok=True) - # Create script file for debugging - script_file = env.ARTIFACT_DIR / f"{step_name_safe}.sh" + # Create script file for debugging in step_scripts directory with index prefix + step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" + step_scripts_dir.mkdir(parents=True, exist_ok=True) + script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" # Log start banner and save script log_caliper_start_banner(command, script_file, step_name.upper()) From 4dd1ee925e6a8ed7cbd5a64d83129b13de3ef643 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 15:02:19 +0200 Subject: [PATCH 0047/1072] [caliper] orchestration: postprocess: make the file paths relative --- projects/caliper/orchestration/postprocess.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 4f0645274..755f7888e 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -373,9 +373,11 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info(f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}") return { "status": "success", - "output_file": str(output_file), + "output_file": relative_path, "completed_at": time.time(), "log_file": log_file, } @@ -449,7 +451,7 @@ def _run_artifacts_to_ai_data( return { "status": "success", "output_file": status_data.get("output_file", str(ai_data_dir)), - "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), + "ai_data_dir": _make_path_relative_to_base(ai_data_dir, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -645,7 +647,7 @@ def _run_kpis_to_csv( return { "status": "success", "kpi_count": status_data.get("kpi_count", 0), - "output_file": str(output_file), + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -728,7 +730,7 @@ def _run_analyse_kpis( if status_data.get("success"): return { "status": "success", - "output_file": str(output_file), + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), "completed_at": time.time(), "log_file": log_file, } @@ -1421,7 +1423,7 @@ def _run_s3_import_step(self, output_dir: Path) -> None: import_dir = output_dir / self.config.s3.import_.output_dir step_result = { "status": "success", - "output_dir": str(import_dir), + "output_dir": _make_path_relative_to_base(import_dir, env.ARTIFACT_DIR), "file_count": status_data.get("file_count", 0), "completed_at": time.time(), } @@ -1491,8 +1493,8 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - # current_kpis_file is now an absolute path (updated from relative path fix) - current_kpis_path = Path(current_kpis_file) + # Convert relative path back to absolute path + current_kpis_path = env.ARTIFACT_DIR / current_kpis_file result = _run_analyse_kpis( postprocess_config=self.config, @@ -1539,7 +1541,9 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and artifacts_to_kpis_step.get("status") == "success" and artifacts_to_kpis_step.get("output_file") ): - kpis_file = output_dir / artifacts_to_kpis_step["output_file"] + stored_output_file = artifacts_to_kpis_step["output_file"] + kpis_file = env.ARTIFACT_DIR / stored_output_file + logger.info(f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}") # Get CSV file from kpis_to_csv step kpis_to_csv_step = self._get_step("kpis_to_csv") @@ -1548,7 +1552,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and kpis_to_csv_step.get("status") == "success" and kpis_to_csv_step.get("output_file") ): - csv_file = output_dir / kpis_to_csv_step["output_file"] + csv_file = env.ARTIFACT_DIR / kpis_to_csv_step["output_file"] # Get AI data directory from artifacts_to_ai_data step ai_data_step = self._get_step("artifacts_to_ai_data") @@ -1558,7 +1562,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and ai_data_step.get("status") == "success" and ai_data_step.get("ai_data_dir") ): - ai_data_dir = output_dir / ai_data_step["ai_data_dir"] + ai_data_dir = env.ARTIFACT_DIR / ai_data_step["ai_data_dir"] logger.info(f"AI data directory set to: {ai_data_dir}") else: logger.warning( @@ -1572,7 +1576,7 @@ def _run_s3_export_step(self, output_dir: Path) -> None: and analyze_step.get("status") in ("success", "warning") and analyze_step.get("output_file") ): - analysis_file = output_dir / analyze_step["output_file"] + analysis_file = env.ARTIFACT_DIR / analyze_step["output_file"] # Create temporary status file for subprocess communication status_file = output_dir / "s3_export_status.yaml" From 99bc553aefa9c7f76243969cf40bc15d07c6e877 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 15:04:00 +0200 Subject: [PATCH 0048/1072] [caliper] orchestration: postprocess: fix the header/footer banner in the logs --- projects/caliper/orchestration/postprocess.py | 160 +++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 755f7888e..6fb243660 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -33,7 +33,6 @@ build_s3_export_command, build_s3_import_command, build_visualize_command, - handle_caliper_output_and_completion, log_caliper_start_banner, ) from projects.caliper.orchestration.postprocess_config import ( @@ -53,6 +52,157 @@ logger = logging.getLogger(__name__) +def _format_command_for_log(command: list[str]) -> str: + """Format command for log file display with only --arguments on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for log file + """ + if not command: + return "" + + # Start with base command parts until we hit options + lines = [] + base_parts = [] + i = 0 + + # Collect all non-flag arguments into base_parts + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle --flag arguments on separate lines + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return " \\\n ".join(lines) + + +def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: + """Write step header to log file. + + Args: + log_file: Path to log file + command: Command that will be executed + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + formatted_command = _format_command_for_log(command) + header = f"""{"=" * 60} +🚀 STARTING {step_name} (fork/exec) +📄 Command: + {formatted_command} +{"=" * 60} + +""" + with open(log_file, "w", encoding="utf-8") as f: + f.write(header) + + +def _write_step_footer_to_log_file( + log_file: Path, result: subprocess.CompletedProcess, step_name: str +) -> None: + """Write step footer to log file. + + Args: + log_file: Path to log file + result: Subprocess result + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + footer = f""" + +{"=" * 60} +🏁 {step_name} COMPLETED +📊 Exit code: {result.returncode} +📄 Log file: {log_file} +{"=" * 60} +""" + with open(log_file, "a", encoding="utf-8") as f: + f.write(footer) + + +def _handle_caliper_output_and_completion_with_header( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + This version appends to the log file instead of overwriting it, since we've + already written a header. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Append output to log file (don't overwrite since we've written header) + with open(log_file, "a", encoding="utf-8") as log_f: + if result.stdout: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + if isinstance(status_data, dict): + for key, value in status_data.items(): + logger.info(" %s: %s", key, value) + else: + logger.info(" %s", status_data) + + logger.info("=" * 60) + + return status_data + + def _execute_caliper_command( command: list[str], step_name: str, @@ -86,6 +236,9 @@ def _execute_caliper_command( # Log start banner and save script log_caliper_start_banner(command, script_file, step_name.upper()) + # Write header to log file + _write_step_header_to_log_file(log_file, command, step_name.upper()) + # Execute command with combined stdout/stderr result = subprocess.run( command, @@ -95,10 +248,13 @@ def _execute_caliper_command( ) # Handle output, status parsing, and log completion banner - status_data = handle_caliper_output_and_completion( + status_data = _handle_caliper_output_and_completion_with_header( result, log_file, status_file, step_name.upper() ) + # Write footer to log file + _write_step_footer_to_log_file(log_file, result, step_name.upper()) + return result, status_data, log_file From fee94ecd2a9c606a29a4a25687cd80498521de1b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:25:35 +0200 Subject: [PATCH 0049/1072] [caliper] engine: visualize: better detect errors --- projects/caliper/engine/visualize.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index f85f6083c..e2947c151 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -89,12 +89,31 @@ def run_visualize( report_group=report_group, config=cfg, ) + + # Check if any report IDs were provided + if not ids: + raise ValueError( + "No report IDs provided. Use --reports with comma-separated report IDs " + "or --report-group with a valid visualize config file." + ) + output_dir.mkdir(parents=True, exist_ok=True) viz = plugin.visualize - return viz( + paths = viz( model, output_dir, ids, report_group, cfg, ) + + # Check for error conditions + if ids and not paths: + # Reports were requested but no files were generated + # This could be due to invalid plot types or other issues + raise ValueError( + f"No visualization files were generated despite requesting report IDs: {ids}. " + f"This may indicate unsupported plot types or other visualization errors." + ) + + return paths From 76e5862ada5b05846e69dbb1859f4ea47d2a0995 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:25:58 +0200 Subject: [PATCH 0050/1072] [guidellm] postprocess: guidellm: plugin: better detect errors --- projects/guidellm/postprocess/guidellm/plugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index 3e4c775c1..f570050df 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -145,9 +145,12 @@ def visualize( return paths # Generate reports using the registry + invalid_reports: list[str] = [] + for report_name in wanted: if report_name not in PLOT_REGISTRY: logger.warning("Unknown report '%s' requested", report_name) + invalid_reports.append(report_name) continue plot_config = PLOT_REGISTRY[report_name] @@ -165,6 +168,14 @@ def visualize( except Exception as e: logger.error("Error generating %s: %s", report_name, e) + # Check for invalid report names and raise error if found + if invalid_reports: + available_reports = list(PLOT_REGISTRY.keys()) + raise ValueError( + f"Invalid report types requested: {sorted(invalid_reports)}. " + f"Available types in this plugin: {sorted(available_reports)}" + ) + return paths def kpi_catalog(self) -> list[dict[str, Any]]: From d194b135a593c80d7452b73d13aeeb4557c942ce Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 16:26:09 +0200 Subject: [PATCH 0051/1072] [skeleton] postprocess: default: plugin: better detect errors --- projects/skeleton/postprocess/default/plugin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 04a10eb16..18b00075f 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -60,6 +60,7 @@ def visualize( output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] wanted = frozenset(report_ids or ()) + invalid_plots: list[str] = [] logger.info(f"Will generate plots for: {sorted(wanted)}") @@ -72,6 +73,15 @@ def visualize( paths.append(path) else: logger.warning(f"Requested plot type '{report_id}' not available in this plugin") + invalid_plots.append(report_id) + + # Check for invalid plot types and raise error if found + if invalid_plots: + available_types = list(self.plots.keys()) + raise ValueError( + f"Invalid plot types requested: {sorted(invalid_plots)}. " + f"Available types in this plugin: {sorted(available_types)}" + ) logger.info( f"Skeleton plugin visualization completed. Generated {len(paths)} files: {paths}" From 801a2f8d9a6a3b35324ca345bcf4af746146f929 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 20:22:36 +0200 Subject: [PATCH 0052/1072] [caliper] orchestration: postprocess: pretty print the ai-eval status --- projects/caliper/orchestration/postprocess.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 6fb243660..b364083f8 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -530,7 +530,9 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) - logger.info(f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}") + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) return { "status": "success", "output_file": relative_path, @@ -1699,7 +1701,9 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ): stored_output_file = artifacts_to_kpis_step["output_file"] kpis_file = env.ARTIFACT_DIR / stored_output_file - logger.info(f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}") + logger.info( + f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}" + ) # Get CSV file from kpis_to_csv step kpis_to_csv_step = self._get_step("kpis_to_csv") @@ -1712,7 +1716,8 @@ def _run_s3_export_step(self, output_dir: Path) -> None: # Get AI data directory from artifacts_to_ai_data step ai_data_step = self._get_step("artifacts_to_ai_data") - logger.info(f"AI data step found: {ai_data_step}") + + logger.info(f"AI data step found:\n{json.dumps(ai_data_step, indent=2)}") if ( ai_data_step and ai_data_step.get("status") == "success" From a2532c75eb540be1fcbd928b9e1409117ebddf5b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 21:30:16 +0200 Subject: [PATCH 0053/1072] [caliper] Final cleanup --- projects/caliper/cli/commands.py | 36 +-- .../caliper/cli/orchestration_entrypoints.py | 210 ------------------ projects/caliper/engine/kpi/analyze.py | 1 - projects/caliper/orchestration/postprocess.py | 40 ++-- .../caliper/orchestration/step_logging.py | 7 - 5 files changed, 47 insertions(+), 247 deletions(-) delete mode 100644 projects/caliper/cli/orchestration_entrypoints.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 583f20b29..3bcafe341 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -8,7 +8,6 @@ import click import yaml -from projects.caliper.cli.orchestration_entrypoints import export_kpis_to_csv_entrypoint from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export @@ -531,8 +530,10 @@ def kpi_csv_export( kpi_record = json.loads(line) kpi_records.append(kpi_record) - # Export to CSV using the entrypoint - result_path = export_kpis_to_csv_entrypoint( + # Export to CSV + from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv + + result_path = export_kpis_to_csv( plugin=plugin, kpi_records=kpi_records, output_path=output, @@ -735,20 +736,29 @@ def analyse_kpis_cmd( output_dir = output.parent - # Run analysis using the entrypoint (without base_dir since we have direct paths) - from projects.caliper.cli.orchestration_entrypoints import analyze_kpis_entrypoint + # Run analysis + from projects.caliper.engine.kpi.analyze import analyze_kpis - result = analyze_kpis_entrypoint( - postprocess_config=postprocess_config_obj, - plugin_module=plugin_module, - base_dir=output_dir, # Use output dir as base since we have absolute paths - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) + try: + result = analyze_kpis( + postprocess_config=postprocess_config_obj, + plugin_module=plugin_module, + base_dir=output_dir, # Use output dir as base since we have absolute paths + output_dir=output_dir, + current_kpis_file=current_kpis_file, + ) + except Exception as e: + # Don't suppress the exception - let it bubble up with full traceback + import sys + import traceback + + print(f"❌ Exception in analyze_kpis: {e}", file=sys.stderr) + print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) + raise # Check if the analysis function returned a proper result if not isinstance(result, dict) or "status" not in result: - raise ValueError(f"Invalid result from analyze_kpis_entrypoint: {result}") + raise ValueError(f"Invalid result from analyze_kpis: {result}") status_data = { "success": result.get("status") == "success", diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py deleted file mode 100644 index 5e319fd22..000000000 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -Entrypoint functions for orchestration to call Caliper core functionality. - -This module provides well-defined interfaces between the orchestration layer -and Caliper's engine/CLI components. -""" - -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any - - -def parse_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, - use_cache: bool = True, -) -> tuple[Any, str]: - """Entrypoint for orchestration to call parse functionality.""" - from projects.caliper.engine.parse import run_parse - - mod_str, plugin = load_plugin_entrypoint(postprocess_config, tree_root, manifest_path) - - model = run_parse( - base_dir=tree_root, - plugin_module=mod_str, - plugin=plugin, - use_cache=use_cache, - ) - - return model, mod_str - - -def run_visualize_entrypoint(**kwargs) -> list[str]: - """Entrypoint for orchestration to call visualize functionality.""" - from projects.caliper.engine.visualize import run_visualize - - return run_visualize(**kwargs) - - -def load_plugin_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, -) -> tuple[str, Any]: - """Entrypoint for orchestration to load plugins.""" - from projects.caliper.engine.load_plugin import load_plugin - from projects.caliper.engine.plugin_config import resolve_plugin_module_string - - mod_str, _manifest = resolve_plugin_module_string( - base_dir=tree_root, - postprocess_config=manifest_path, - cli_plugin=postprocess_config.plugin_module, - ) - - return mod_str, load_plugin(mod_str) - - -def s3_import_entrypoint( - postprocess_config, - output_dir: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 import functionality.""" - - if not postprocess_config.s3.import_.enabled: - return {"status": "disabled", "reason": "s3_import disabled", "completed_at": time.time()} - - try: - from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params - - # Build import prefix using instance + directory - from projects.caliper.cli.s3_export import build_s3_prefix - - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.import_ - - import_prefix = build_s3_prefix( - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - trailing_slash=False, - ) - import_dir = output_dir / s3_config.output_dir - - # Use explicit parameters instead of config dict - result = run_s3_import_with_explicit_params( - bucket=s3_parent_config.bucket, - prefix=import_prefix, - output_dir=import_dir, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - return result - - except Exception as e: - import logging - - logger = logging.getLogger(__name__) - logger.exception("S3 import failed") - return {"status": "failed", "error": str(e), "completed_at": time.time()} - - -def s3_export_entrypoint( - postprocess_config, - output_dir: Path, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 export functionality with explicit paths.""" - from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths - - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.export - - return run_s3_export_with_explicit_paths( - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, - bucket=s3_parent_config.bucket, - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - upload_id=s3_config.upload_id, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - dry_run=s3_config.dry_run, - ) - - -def get_kpi_functions_entrypoint(plugin_module) -> dict: - """Entrypoint for orchestration to get KPI function metadata.""" - from projects.caliper.engine.kpi.decorators import get_kpi_functions - - try: - plugin_module_obj = __import__(plugin_module, fromlist=[""]) - return get_kpi_functions(plugin_module_obj) - except (ImportError, AttributeError): - return {} - - -def analyze_kpis_entrypoint( - postprocess_config, - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call KPI analysis.""" - from projects.caliper.engine.kpi.analyze import analyze_kpis - - try: - result = analyze_kpis( - postprocess_config=postprocess_config, - plugin_module=plugin_module, - base_dir=base_dir, - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) - return result - except Exception as e: - # Don't suppress the exception - let it bubble up with full traceback - import sys - import traceback - - print(f"❌ Exception in analyze_kpis_entrypoint: {e}", file=sys.stderr) - print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) - raise - - -def kpi_generate_entrypoint( - base_dir: Path, - plugin_module: str, - plugin: Any, - output: Path, -) -> None: - """Entrypoint for orchestration to generate KPIs.""" - from projects.caliper.engine.kpi.generate import run_kpi_generate - - return run_kpi_generate( - base_dir=base_dir, - plugin_module=plugin_module, - plugin=plugin, - output=output, - use_cache=True, - cache_path=None, - ) - - -def export_kpis_to_csv_entrypoint( - plugin, - kpi_records: list[dict[str, Any]], - output_path: Path, - include_header_comments: bool = True, -) -> str: - """Entrypoint for orchestration to export KPIs to CSV.""" - from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv - - return export_kpis_to_csv( - plugin=plugin, - kpi_records=kpi_records, - output_path=output_path, - include_header_comments=include_header_comments, - ) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 40ae18680..825762b62 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -156,7 +156,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Call the core analysis function exit_code = run_kpi_analysis( current_kpi_file=current_kpis_file, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index b364083f8..c0a594075 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -19,11 +19,6 @@ from pydantic import ValidationError -from projects.caliper.cli.orchestration_entrypoints import ( - get_kpi_functions_entrypoint, - load_plugin_entrypoint, - parse_entrypoint, -) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, build_analyse_kpis_command, @@ -329,9 +324,6 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() -# _load_plugin function removed - now using load_plugin_entrypoint - - def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: """ Transform flat KPI list into hierarchical JSON structure. @@ -355,7 +347,13 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) # Get KPI function metadata from the plugin module - kpi_functions = get_kpi_functions_entrypoint(model.plugin_module) + from projects.caliper.engine.kpi.decorators import get_kpi_functions + + try: + plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) + kpi_functions = get_kpi_functions(plugin_module_obj) + except (ImportError, AttributeError): + kpi_functions = {} for kpi in kpis: run_id = kpi.get("run_id", "unknown") @@ -1352,14 +1350,24 @@ def _run_kpi_and_ai_data_steps(self) -> None: output_dir.mkdir(parents=True, exist_ok=True) # Load plugin and model - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, + from projects.caliper.engine.load_plugin import load_plugin + from projects.caliper.engine.parse import run_parse + from projects.caliper.engine.plugin_config import resolve_plugin_module_string + + # Load plugin + mod_str, _manifest = resolve_plugin_module_string( + base_dir=self.tree_root, + postprocess_config=self.manifest_path, + cli_plugin=self.config.plugin_module, ) - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path + plugin = load_plugin(mod_str) + + # Parse model + model = run_parse( + base_dir=self.tree_root, + plugin_module=mod_str, + plugin=plugin, + use_cache=not self.config.parse.no_cache, ) # KPI JSON generation diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 5354c48e0..66cd83137 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -11,7 +11,6 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from typing import Any # Thread-local storage for step-specific handlers _step_local_handlers = threading.local() @@ -235,9 +234,3 @@ def cleanup_step_logging(): if hasattr(_step_local_handlers, "file_handler"): _step_local_handlers.file_handler.close() del _step_local_handlers.file_handler - - - - - - From 2ac95f6a9517812528042445560675a9af62f307 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 22:30:56 +0200 Subject: [PATCH 0054/1072] [rhoai] toolbox: wait_datasciencecluster_ready: about if the DSC isn't found --- .../rhoai/toolbox/wait_datasciencecluster_ready/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py index 27a1d1631..cd1a7f79f 100644 --- a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py +++ b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py @@ -73,6 +73,14 @@ def wait_for_datasciencecluster_ready(args, ctx): log_stdout=True, # Show the output check=False, ) + + # Abort if the DataScienceCluster is not found + if result.returncode != 0: + error_msg = f"DataScienceCluster {args.datasciencecluster_name} not found in namespace {args.namespace}" + if result.stderr: + error_msg += f": {result.stderr.strip()}" + raise RuntimeError(error_msg) + phase = result.stdout.strip() if result.stdout else None if phase == "Ready": From fedcd318f0c1f17a5a10147303400cde8fd1c4e3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 11 Jul 2026 08:30:49 +0200 Subject: [PATCH 0055/1072] [core] ci_entrypoint: prepare_ci: improve the status message --- projects/core/ci_entrypoint/prepare_ci.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 076d3b01e..11dad7411 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -783,12 +783,17 @@ def postchecks( """ artifact_dir = os.environ.get("ARTIFACT_DIR") + # Build command string for display + command_str = f"{project} {operation}" + if args: + command_str += f" {' '.join(args)}" + if not artifact_dir: # No artifact dir, just return simple status return ( - f"✅ {project} {operation} completed successfully" + f"✅ {command_str} completed successfully" if finish_reason == FinishReason.SUCCESS - else f"❌ {project} {operation} failed" + else f"❌ {command_str} failed" ) artifact_path = Path(artifact_dir) @@ -821,8 +826,8 @@ def postchecks( if finish_reason != FinishReason.SUCCESS or ( failures_file.exists() and failures_file.stat().st_size > 0 ): - status = f"❌ Execution of '{project} {operation}' failed after{duration_str}." + status = f"❌ Execution of '{command_str}' failed after{duration_str}." else: - status = f"✅ Execution of '{project} {operation}' succeeded after{duration_str}." + status = f"✅ Execution of '{command_str}' succeeded after{duration_str}." return status From 405ba90c557dd7f8072b8e3790ee0b1ef4ec34b9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 08:51:32 +0200 Subject: [PATCH 0056/1072] Revert "fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs" This reverts commit a55a40a7b261e053018bacdf910dfc9d57ab7c60. --- .../base/workflows/task-forge-step.yaml | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index 72e58e9c0..dcb29b30c 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,8 +13,6 @@ spec: type: string - name: kubeconfig-secret type: string - default: "fournos-clusterless-placeholder" - description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -33,8 +31,6 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig - - name: KUBECONFIG_SECRET_PARAM - value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -77,24 +73,7 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - # Check if we have a kubeconfig for target cluster - if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then - echo "Setting up target cluster kubeconfig..." - export KUBECONFIG=$TARGET_KUBECONFIG - export CLUSTERLESS_MODE="false" - echo "Target cluster info:" - kubectl cluster-info - else - if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then - echo "Running in clusterless mode - no target cluster kubeconfig" - else - echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" - echo "Running in clusterless mode as fallback" - fi - export CLUSTERLESS_MODE="true" - echo "Hub cluster info:" - kubectl cluster-info - fi + export KUBECONFIG=$TARGET_KUBECONFIG # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -147,4 +126,3 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) - optional: true From d6bbedabceda702adff37cbe84adab682d0b6f4a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 09:20:19 +0200 Subject: [PATCH 0057/1072] [foreign_testing] orchestration: foreign_testing: fix --- projects/foreign_testing/orchestration/foreign_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/foreign_testing/orchestration/foreign_testing.py b/projects/foreign_testing/orchestration/foreign_testing.py index e4451c05d..fd92e891c 100644 --- a/projects/foreign_testing/orchestration/foreign_testing.py +++ b/projects/foreign_testing/orchestration/foreign_testing.py @@ -3,7 +3,7 @@ import pathlib import shutil -from projects.core.ci_entrypoint import run_ci +from projects.core.ci_entrypoint import run_common from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def submit(project_path=None): launch_args = [*["--project-source", str(project_path)], *launch_args] logger.info(f"Submitting deployment with args: {launch_args}") - run_ci.execute_project_operation( + run_common.execute_project_operation( project, operation, launch_args, From 2006a4f2b4f9939126af7c5e01a8bdbf34b476fa Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 10:23:56 +0100 Subject: [PATCH 0058/1072] feat: Add ICSP for staging images Signed-off-by: Alberto Perdomo --- .../orchestration/config.d/platform.yaml | 4 + .../manifests/quay-registry-icsp.yaml | 9 -- .../manifests/rhoai-registry-icsp.yaml | 15 ++ projects/llm_d/orchestration/prepare_phase.py | 2 +- projects/llm_d/tests/test_profiles.py | 140 ++++++++++++++++- projects/rhoai/library/deploy.py | 146 +++++++++++++++--- vaults/psap-forge-staging-image-pull.yaml | 10 ++ 7 files changed, 287 insertions(+), 39 deletions(-) delete mode 100644 projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml create mode 100644 projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml create mode 100644 vaults/psap-forge-staging-image-pull.yaml diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index f9b2975e2..bf8ec298b 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -59,6 +59,10 @@ rhoai: vault: name: psap-rhoai-rc content: rhoai_rc.secret + staging_pull_secret: + vault: + name: psap-forge-staging-image-pull + content: .dockerconfigjson namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: diff --git a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml deleted file mode 100644 index d18702599..000000000 --- a/projects/llm_d/orchestration/manifests/quay-registry-icsp.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy -metadata: - name: quay-registry -spec: - repositoryDigestMirrors: - - source: registry.redhat.io/rhoai - mirrors: - - quay.io/rhoai diff --git a/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml new file mode 100644 index 000000000..591e1e378 --- /dev/null +++ b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml @@ -0,0 +1,15 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: rhoai-registry +spec: + repositoryDigestMirrors: + - source: registry.redhat.io/rhoai + mirrors: + - quay.io/rhoai + - source: registry.redhat.io/rhaii + mirrors: + - registry.stage.redhat.io/rhaii + - source: registry.redhat.io/rhaii-early-access + mirrors: + - registry.stage.redhat.io/rhaii-early-access diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 3e3b63047..930f35e6a 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -40,7 +40,7 @@ ) logger = logging.getLogger(__name__) -RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "quay-registry-icsp.yaml" +RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "rhoai-registry-icsp.yaml" def verify_oc_access() -> None: diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 97a03cdfb..1d5f197c1 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import json from pathlib import Path import pytest @@ -259,7 +261,18 @@ def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deploy "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", "display_name": "Red Hat OpenShift AI", "publisher": "RHOAI Development Catalog", - "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, }, "namespace": "redhat-ods-applications", "datasciencecluster_name": "default-dsc", @@ -325,12 +338,20 @@ def _fake_get_vault_content_path(vault_name: str, content_name: str): path = rhoai_deploy.custom_catalog_pull_secret_path( { - "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, } ) assert path == Path("/tmp/rhoai_rc.secret") - assert captured == {"vault_name": "psap-rhoai-rc", "content_name": "rhoai_rc.secret"} + assert captured == { + "vault_name": "psap-rhoai-rc", + "content_name": "rhoai_rc.secret", + } def test_wait_for_rhoai_pull_secret_ready_treats_empty_mcp_status_as_not_ready( @@ -357,13 +378,124 @@ def _fake_oc(*args, **kwargs): monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) - monkeypatch.setattr(rhoai_deploy, "_decode_pull_secret", lambda _secret: "quay.io/rhoai") + monkeypatch.setattr( + rhoai_deploy, + "_decode_pull_secret", + lambda _secret: ( + "quay.io/rhoai registry.stage.redhat.io/rhaii " + "registry.stage.redhat.io/rhaii-early-access" + ), + ) monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) +def test_prepare_rhoai_pull_secret_merges_dockerconfigjson_registry_auths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _init_project_config() + + rc_pull_secret_path = tmp_path / "rhoai_rc.secret" + rc_pull_secret_path.write_text( + "rhoai+rhoai_external_readonly_bot:token-c", + encoding="utf-8", + ) + + staging_pull_secret_path = tmp_path / ".dockerconfigjson" + staging_pull_secret_path.write_text( + json.dumps( + { + "auths": { + "registry.stage.redhat.io/rhaii": {"auth": "token-a"}, + "registry.stage.redhat.io/rhaii-early-access": {"auth": "token-b"}, + } + } + ), + encoding="utf-8", + ) + + current_secret = { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps({"auths": {}}).encode("utf-8")).decode( + "utf-8" + ) + } + } + merged_payload: dict[str, object] = {} + + monkeypatch.setattr( + rhoai_deploy.vault, + "get_vault_content_path", + lambda vault_name, content_name: ( + rc_pull_secret_path + if (vault_name, content_name) == ("psap-rhoai-rc", "rhoai_rc.secret") + else staging_pull_secret_path + ), + ) + monkeypatch.setattr(rhoai_deploy, "oc_get_json", lambda *args, **kwargs: current_secret) + + def _fake_oc(*args, **kwargs): + if args[:2] == ("set", "data"): + from_file = next((arg for arg in args if str(arg).startswith("--from-file=")), "") + payload_path = Path(from_file.rsplit("=", 1)[1]) + merged_payload.update(json.loads(payload_path.read_text(encoding="utf-8"))) + return type("Result", (), {"stdout": "", "returncode": 0})() + if args[:2] == ("registry", "login"): + registry = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--registry=") + ) + auth_basic = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--auth-basic=") + ) + temp_path = Path( + next( + arg + for arg in args + if not str(arg).startswith("--") and arg != "registry" and arg != "login" + ) + ) + current = json.loads(temp_path.read_text(encoding="utf-8")) + current.setdefault("auths", {})[registry] = {"auth": auth_basic} + temp_path.write_text(json.dumps(current, indent=2, sort_keys=True), encoding="utf-8") + return type("Result", (), {"stdout": "", "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy, "wait_for_rhoai_pull_secret_ready", lambda **kwargs: None) + + rhoai_deploy.prepare_rhoai_pull_secret( + rhoai_deploy.RhoaiCustomCatalogConfig.model_validate( + { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, + } + ) + ) + + assert ( + merged_payload["auths"]["quay.io/rhoai"]["auth"] + == "rhoai+rhoai_external_readonly_bot:token-c" + ) + assert merged_payload["auths"]["registry.stage.redhat.io/rhaii"]["auth"] == "token-a" + assert ( + merged_payload["auths"]["registry.stage.redhat.io/rhaii-early-access"]["auth"] == "token-b" + ) + + def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: _init_project_config() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index fa4c6203c..233479d87 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import json import logging import tempfile import time @@ -20,7 +21,11 @@ RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" RHOAI_PULL_SECRET_NAME = "pull-secret" -RHOAI_REGISTRY = "quay.io/rhoai" +RHOAI_CATALOG_REGISTRIES = ("quay.io/rhoai",) +RHOAI_REGISTRIES = ( + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", +) class _RhoaiCustomCatalogPullSecretVaultConfig(BaseModel): @@ -52,6 +57,7 @@ class RhoaiCustomCatalogConfig(BaseModel): display_name: str | None = None publisher: str | None = None pull_secret: _RhoaiCustomCatalogPullSecretConfig + staging_pull_secret: _RhoaiCustomCatalogPullSecretConfig | None = None class RhoaiOperatorConfig(BaseModel): @@ -149,6 +155,77 @@ def _decode_pull_secret(secret_data: dict[str, Any]) -> str: return base64.b64decode(encoded).decode("utf-8") +def _read_vault_pull_secret(pull_secret_path: Path) -> str | dict[str, Any]: + raw_content = pull_secret_path.read_text(encoding="utf-8").strip() + if not raw_content: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + try: + parsed = json.loads(raw_content) + except json.JSONDecodeError: + return raw_content + + if isinstance(parsed, dict) and "auths" in parsed: + return parsed + + return raw_content + + +def _apply_vault_pull_secret( + *, + current_secret: dict[str, Any], + pull_secret_path: Path, + registries: tuple[str, ...], + temp_path: Path, +) -> dict[str, Any]: + pull_secret_payload = _read_vault_pull_secret(pull_secret_path) + + if isinstance(pull_secret_payload, dict): + merged_secret = _merge_pull_secret_auths(current_secret, pull_secret_payload) + temp_path.write_text(json.dumps(merged_secret, indent=2, sort_keys=True), encoding="utf-8") + return merged_secret + + for registry in registries: + oc( + "registry", + "login", + f"--registry={registry}", + f"--auth-basic={pull_secret_payload}", + "--to", + str(temp_path), + ) + return json.loads(temp_path.read_text(encoding="utf-8")) + + +def _current_pull_secret_json() -> dict[str, Any]: + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + try: + return json.loads(decoded_secret) + except json.JSONDecodeError as exc: + raise RuntimeError("openshift-config/pull-secret is not valid dockerconfigjson") from exc + + +def _registries_present( + decoded_secret: str, registries: tuple[str, ...] = RHOAI_REGISTRIES +) -> bool: + return all(registry in decoded_secret for registry in registries) + + +def _merge_pull_secret_auths( + current_secret: dict[str, Any], pull_secret_payload: dict[str, Any] +) -> dict[str, Any]: + merged_secret = dict(current_secret) + merged_auths = dict(current_secret.get("auths", {})) + merged_auths.update(pull_secret_payload.get("auths", {})) + merged_secret["auths"] = merged_auths + return merged_secret + + def wait_for_rhoai_pull_secret_ready( *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 ) -> None: @@ -163,8 +240,8 @@ def wait_for_rhoai_pull_secret_ready( namespace=RHOAI_PULL_SECRET_NAMESPACE, ) decoded = _decode_pull_secret(secret) - if RHOAI_REGISTRY not in decoded: - raise RuntimeError(f"{RHOAI_REGISTRY} not yet present in pull secret") + if not _registries_present(decoded, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): + raise RuntimeError("required RHOAI registries are not yet present in pull secret") mcp_status = oc( "get", @@ -180,7 +257,7 @@ def wait_for_rhoai_pull_secret_ready( logger.info( "RHOAI pull secret now contains %s and machine config pools are updated", - RHOAI_REGISTRY, + ", ".join(RHOAI_REGISTRIES), ) return except RuntimeError as exc: @@ -189,7 +266,7 @@ def wait_for_rhoai_pull_secret_ready( time.sleep(poll_interval_seconds) raise RuntimeError( - f"Timed out waiting for {RHOAI_REGISTRY} pull secret propagation: {last_error}" + f"Timed out waiting for RHOAI registries pull secret propagation: {last_error}" ) @@ -204,35 +281,54 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " f"'{custom_catalog.pull_secret.vault.name}'" ) - auth_basic = pull_secret_path.read_text(encoding="utf-8").strip() - if not auth_basic: - raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") - current_secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded_secret = _decode_pull_secret(current_secret) + current_secret = _current_pull_secret_json() + current_secret_text = json.dumps(current_secret, sort_keys=True) - if RHOAI_REGISTRY in decoded_secret: - logger.info("RHOAI registry %s already present in cluster pull secret", RHOAI_REGISTRY) + required_registries = (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES) + + if _registries_present(current_secret_text, required_registries): + logger.info( + "RHOAI registries already present in cluster pull secret: %s", + ", ".join(required_registries), + ) return - logger.info("Adding %s to cluster pull secret from %s", RHOAI_REGISTRY, pull_secret_path) + logger.info( + "Adding %s to cluster pull secret from %s", + ", ".join(required_registries), + pull_secret_path, + ) with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: - temp_file.write(decoded_secret) + temp_file.write(json.dumps(current_secret, indent=2, sort_keys=True)) temp_path = Path(temp_file.name) try: - oc( - "registry", - "login", - f"--registry={RHOAI_REGISTRY}", - f"--auth-basic={auth_basic}", - "--to", - str(temp_path), + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=pull_secret_path, + registries=RHOAI_CATALOG_REGISTRIES, + temp_path=temp_path, ) + + if custom_catalog.staging_pull_secret is not None: + staging_pull_secret_path = vault.get_vault_content_path( + custom_catalog.staging_pull_secret.vault.name, + custom_catalog.staging_pull_secret.vault.content, + ) + if staging_pull_secret_path is None: + raise RuntimeError( + "RHOAI staging pull secret content " + f"'{custom_catalog.staging_pull_secret.vault.content}' was not found in vault " + f"'{custom_catalog.staging_pull_secret.vault.name}'" + ) + + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=staging_pull_secret_path, + registries=RHOAI_REGISTRIES, + temp_path=temp_path, + ) oc( "set", "data", diff --git a/vaults/psap-forge-staging-image-pull.yaml b/vaults/psap-forge-staging-image-pull.yaml new file mode 100644 index 000000000..5979f3b4d --- /dev/null +++ b/vaults/psap-forge-staging-image-pull.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_FORGE_STAGING_IMAGE_PULL_SECRET_PATH +description: | + Vault containing registry credentials for staging image pulling + +content: + .dockerconfigjson: + description: | + Docker registry auth config used to extend the cluster pull secret + sample: | + {"auths":{"registry.stage.redhat.io":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii-early-access":{"auth":"base64-encoded-credentials"}}} From dfe4f004dc8f878cb1c35103d6441285f0ba75b1 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 11:57:07 +0100 Subject: [PATCH 0059/1072] fix: Mount staging vault secret Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/ci.py | 8 ++++++-- projects/llm_d/tests/test_profiles.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index f8611771c..b3748f679 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -28,6 +28,10 @@ from projects.llm_d.orchestration.test_phase import run as test_toolbox_run logger = logging.getLogger(__name__) +RHOAI_CUSTOM_CATALOG_VAULTS = [ + "psap-rhoai-rc", + "psap-forge-staging-image-pull", +] def init(): @@ -41,7 +45,7 @@ def list_vaults() -> list[str]: """List all vaults (includes both mandatory and optional).""" all_vaults = vault.phase_vault_list_all() if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): - return [*all_vaults, "psap-rhoai-rc"] + return [*all_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] return all_vaults @@ -59,7 +63,7 @@ def init_vaults_for_phase(phase: str) -> None: if phase == "prepare" and config.project.get_config( "platform.rhoai.custom_catalog.enabled", False ): - optional_vaults = [*optional_vaults, "psap-rhoai-rc"] + optional_vaults = [*optional_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 1d5f197c1..20b565954 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -201,17 +201,19 @@ def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided assert runtime_config.get_benchmark_keys() == ["short"] -def test_list_vaults_only_includes_rhoai_rc_vault_for_custom_catalog_runs() -> None: +def test_list_vaults_only_includes_rhoai_custom_catalog_vaults_for_custom_catalog_runs() -> None: _init_project_config() core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) assert "psap-rhoai-rc" not in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" not in llmd_ci.list_vaults() core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) assert "psap-rhoai-rc" in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" in llmd_ci.list_vaults() -def test_prepare_phase_adds_rhoai_rc_vault_only_for_custom_catalog_runs( +def test_prepare_phase_adds_rhoai_custom_catalog_vaults_only_for_custom_catalog_runs( monkeypatch: pytest.MonkeyPatch, ) -> None: _init_project_config() @@ -230,7 +232,9 @@ def _fake_init(**kwargs): llmd_ci.init_vaults_for_phase("prepare") assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] + assert "psap-forge-staging-image-pull" not in calls[0]["optional_vaults"] assert "psap-rhoai-rc" in calls[1]["optional_vaults"] + assert "psap-forge-staging-image-pull" in calls[1]["optional_vaults"] def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( From 1444aac498555f8ac0183f4b08e9a87c50101c4f Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 12:30:56 +0100 Subject: [PATCH 0060/1072] fix: Loose substring check Signed-off-by: Alberto Perdomo --- projects/llm_d/tests/test_profiles.py | 40 +++++++++++++++++++++------ projects/rhoai/library/deploy.py | 32 ++++++++++++--------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 20b565954..6b9c948d4 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -373,7 +373,19 @@ def _fake_sleep(_seconds: float) -> None: clock["value"] += 0.1 def _fake_oc_get_json(*args, **kwargs): - return {"data": {".dockerconfigjson": "unused"}} + payload = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + return { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps(payload).encode("utf-8")).decode( + "utf-8" + ) + } + } def _fake_oc(*args, **kwargs): if args[:2] == ("get", "mcp"): @@ -382,20 +394,30 @@ def _fake_oc(*args, **kwargs): monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) - monkeypatch.setattr( - rhoai_deploy, - "_decode_pull_secret", - lambda _secret: ( - "quay.io/rhoai registry.stage.redhat.io/rhaii " - "registry.stage.redhat.io/rhaii-early-access" - ), - ) monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) +def test_registries_present_accepts_parent_registry_auth_entries() -> None: + pull_secret = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + + assert rhoai_deploy._registries_present( + pull_secret, + ( + "quay.io/rhoai", + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", + ), + ) + + def test_prepare_rhoai_pull_secret_merges_dockerconfigjson_registry_auths( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 233479d87..91c147547 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -210,10 +210,24 @@ def _current_pull_secret_json() -> dict[str, Any]: raise RuntimeError("openshift-config/pull-secret is not valid dockerconfigjson") from exc +def _normalize_registry_key(registry: str) -> str: + return registry.removeprefix("https://").removeprefix("http://").rstrip("/") + + +def _auth_entry_covers_registry(auth_key: str, required_registry: str) -> bool: + auth_key = _normalize_registry_key(auth_key) + required_registry = _normalize_registry_key(required_registry) + return auth_key == required_registry or required_registry.startswith(f"{auth_key}/") + + def _registries_present( - decoded_secret: str, registries: tuple[str, ...] = RHOAI_REGISTRIES + pull_secret: dict[str, Any], registries: tuple[str, ...] = RHOAI_REGISTRIES ) -> bool: - return all(registry in decoded_secret for registry in registries) + auth_keys = tuple(pull_secret.get("auths", {}).keys()) + return all( + any(_auth_entry_covers_registry(auth_key, registry) for auth_key in auth_keys) + for registry in registries + ) def _merge_pull_secret_auths( @@ -234,15 +248,9 @@ def wait_for_rhoai_pull_secret_ready( while time.monotonic() < deadline: try: - secret = oc_get_json( - "secret", - name=RHOAI_PULL_SECRET_NAME, - namespace=RHOAI_PULL_SECRET_NAMESPACE, - ) - decoded = _decode_pull_secret(secret) - if not _registries_present(decoded, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): + pull_secret = _current_pull_secret_json() + if not _registries_present(pull_secret, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): raise RuntimeError("required RHOAI registries are not yet present in pull secret") - mcp_status = oc( "get", "mcp", @@ -283,11 +291,9 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) current_secret = _current_pull_secret_json() - current_secret_text = json.dumps(current_secret, sort_keys=True) - required_registries = (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES) - if _registries_present(current_secret_text, required_registries): + if _registries_present(current_secret, required_registries): logger.info( "RHOAI registries already present in cluster pull secret: %s", ", ".join(required_registries), From 4433216b9e185b27926d0624212b3164bfc8da27 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 13 Jul 2026 15:02:53 +0100 Subject: [PATCH 0061/1072] fix: Wait for serving control plane before deploying llmisvc Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/test_phase.py | 58 +++++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f16034aad..f3716dd3e 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -2,11 +2,13 @@ import logging import sys +import time from pathlib import Path from typing import Any from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier +from projects.core.dsl.utils.k8s import oc from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -24,6 +26,13 @@ from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) +SERVING_CONTROL_PLANE_DEPLOYMENTS = ( + "kserve-controller-manager", + "llmisvc-controller-manager", + "model-serving-api", + "odh-model-controller", +) +SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 def create_test_labels() -> None: @@ -177,10 +186,13 @@ def deploy_inference_service() -> str: # Step 1: Ensure model cache is ready _prepare_model_cache() - # Step 2: Build and write inference service manifest + # Step 2: Wait for the serving control plane to settle before creating the service. + _wait_for_serving_control_plane_ready() + + # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() - # Step 3: Deploy the service and wait for endpoint + # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) endpoint_url = deploy_llmisvc.run( namespace=namespace, @@ -204,6 +216,48 @@ def _prepare_model_cache() -> None: prepare_model_cache() +def _wait_for_serving_control_plane_ready() -> None: + from projects.llm_d.orchestration import runtime_config + + rhoai_namespace = runtime_config.get_platform_config()["rhoai"]["namespace"] + + for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: + result = oc( + "wait", + "--for=condition=Available", + "--timeout=300s", + f"deployment/{deployment_name}", + "-n", + rhoai_namespace, + check=False, + ) + if result.returncode != 0: + status_result = oc( + "get", + "deployments", + "-n", + rhoai_namespace, + "-o", + "wide", + check=False, + ) + error_msg = ( + f"Timeout waiting for serving control plane deployment " + f"{deployment_name} in {rhoai_namespace}" + ) + if status_result.returncode == 0: + error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" + raise RuntimeError(error_msg) + + # The webhook deployments can report Available before leader election and informer + # startup finish, which makes the first LLMInferenceService create race the webhook. + logger.info( + "Waiting %ss for serving control plane stabilization", + SERVING_CONTROL_PLANE_STABILIZATION_SECONDS, + ) + time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) + + def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" from projects.llm_d.orchestration import runtime_config From d3ff07e70099d9fc6ff6cf37519f5d284f7e8b72 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Tue, 14 Jul 2026 16:36:02 +0100 Subject: [PATCH 0062/1072] chore: Fix review comments Signed-off-by: Alberto Perdomo --- .../toolbox/deploy_custom_catalog/main.py | 1 - .../kserve/toolbox/wait_kserve_ready/main.py | 81 +++++++++++++++++++ projects/llm_d/orchestration/test_phase.py | 55 +------------ 3 files changed, 84 insertions(+), 53 deletions(-) create mode 100644 projects/kserve/toolbox/wait_kserve_ready/main.py diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 641184cb8..2b70724f0 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -41,7 +41,6 @@ def setup_directories(args, ctx): shell.mkdir("src") shell.mkdir("artifacts") ctx.display_name = args.display_name or args.catalog_source_name - ctx.publisher = args.publisher or "" return f"Prepared directories for {ctx.display_name}" diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py new file mode 100644 index 000000000..11e5ee1ef --- /dev/null +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import time + +from projects.core.dsl import entrypoint, execute_tasks, task +from projects.core.dsl.utils.k8s import oc + +SERVING_CONTROL_PLANE_DEPLOYMENTS = ( + "kserve-controller-manager", + "llmisvc-controller-manager", + "model-serving-api", + "odh-model-controller", +) +SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 + + +@entrypoint +def run(*, namespace: str) -> str: + """ + Wait for the KServe serving control plane deployments to be ready. + + Args: + namespace: Namespace where the serving control plane is deployed + """ + + task_args = {"namespace": namespace} + execute_tasks(task_args) + + return f"Serving control plane is ready in {namespace}" + + +@task +def wait_for_deployments(args, ctx): + """Wait for all serving control plane deployments to be available""" + + for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: + result = oc( + "wait", + "--for=condition=Available", + "--timeout=300s", + f"deployment/{deployment_name}", + "-n", + args.namespace, + check=False, + ) + if result.returncode != 0: + status_result = oc( + "get", + "deployments", + "-n", + args.namespace, + "-o", + "wide", + check=False, + ) + error_msg = ( + f"Timeout waiting for serving control plane deployment " + f"{deployment_name} in {args.namespace}" + ) + if status_result.returncode == 0: + error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" + raise RuntimeError(error_msg) + + return "All serving control plane deployments are available" + + +@task +def wait_for_stabilization(args, ctx): + """Wait for serving control plane to stabilize after becoming available""" + + # The webhook deployments can report Available before leader election and informer + # startup finish, which makes the first LLMInferenceService create race the webhook. + time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) + + return f"Waited {SERVING_CONTROL_PLANE_STABILIZATION_SECONDS}s for stabilization" + + +if __name__ == "__main__": + run.main() diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f3716dd3e..2ef047ef2 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -2,13 +2,11 @@ import logging import sys -import time from pathlib import Path from typing import Any from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier -from projects.core.dsl.utils.k8s import oc from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -18,6 +16,7 @@ from projects.guidellm.toolbox.run_smoke_request import main as run_smoke_request_command from projects.kserve.toolbox.capture_llmisvc_state import main as capture_llmisvc_state from projects.kserve.toolbox.deploy_llmisvc import main as deploy_llmisvc +from projects.kserve.toolbox.wait_kserve_ready import main as wait_kserve_ready from projects.llm_d.orchestration.prepare_phase import prepare_model_cache from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, @@ -26,13 +25,6 @@ from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) -SERVING_CONTROL_PLANE_DEPLOYMENTS = ( - "kserve-controller-manager", - "llmisvc-controller-manager", - "model-serving-api", - "odh-model-controller", -) -SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 def create_test_labels() -> None: @@ -187,7 +179,8 @@ def deploy_inference_service() -> str: _prepare_model_cache() # Step 2: Wait for the serving control plane to settle before creating the service. - _wait_for_serving_control_plane_ready() + rhoai_namespace = platform["rhoai"]["namespace"] + wait_kserve_ready.run(namespace=rhoai_namespace) # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() @@ -216,48 +209,6 @@ def _prepare_model_cache() -> None: prepare_model_cache() -def _wait_for_serving_control_plane_ready() -> None: - from projects.llm_d.orchestration import runtime_config - - rhoai_namespace = runtime_config.get_platform_config()["rhoai"]["namespace"] - - for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: - result = oc( - "wait", - "--for=condition=Available", - "--timeout=300s", - f"deployment/{deployment_name}", - "-n", - rhoai_namespace, - check=False, - ) - if result.returncode != 0: - status_result = oc( - "get", - "deployments", - "-n", - rhoai_namespace, - "-o", - "wide", - check=False, - ) - error_msg = ( - f"Timeout waiting for serving control plane deployment " - f"{deployment_name} in {rhoai_namespace}" - ) - if status_result.returncode == 0: - error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" - raise RuntimeError(error_msg) - - # The webhook deployments can report Available before leader election and informer - # startup finish, which makes the first LLMInferenceService create race the webhook. - logger.info( - "Waiting %ss for serving control plane stabilization", - SERVING_CONTROL_PLANE_STABILIZATION_SECONDS, - ) - time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) - - def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" from projects.llm_d.orchestration import runtime_config From a06b214ce5349206cb5d7de4925d2bce5a7eaaac Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 09:15:16 +0100 Subject: [PATCH 0063/1072] fix: Dry-run instead of time.sleep() Signed-off-by: Alberto Perdomo --- .../kserve/toolbox/wait_kserve_ready/main.py | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py index 11e5ee1ef..426cf2355 100644 --- a/projects/kserve/toolbox/wait_kserve_ready/main.py +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -2,9 +2,9 @@ from __future__ import annotations -import time +import yaml -from projects.core.dsl import entrypoint, execute_tasks, task +from projects.core.dsl import entrypoint, execute_tasks, retry, task from projects.core.dsl.utils.k8s import oc SERVING_CONTROL_PLANE_DEPLOYMENTS = ( @@ -13,7 +13,6 @@ "model-serving-api", "odh-model-controller", ) -SERVING_CONTROL_PLANE_STABILIZATION_SECONDS = 45 @entrypoint @@ -66,15 +65,46 @@ def wait_for_deployments(args, ctx): return "All serving control plane deployments are available" +@retry(attempts=40, delay=5, backoff=1.0) @task -def wait_for_stabilization(args, ctx): - """Wait for serving control plane to stabilize after becoming available""" - - # The webhook deployments can report Available before leader election and informer - # startup finish, which makes the first LLMInferenceService create race the webhook. - time.sleep(SERVING_CONTROL_PLANE_STABILIZATION_SECONDS) - - return f"Waited {SERVING_CONTROL_PLANE_STABILIZATION_SECONDS}s for stabilization" +def probe_webhook_ready(args, ctx): + """Probe the admission webhook with a dry-run apply to confirm it is serving""" + + probe_manifest = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "LLMInferenceService", + "metadata": { + "name": "forge-webhook-probe", + "namespace": args.namespace, + }, + "spec": { + "model": { + "uri": "hf://probe/model", + "name": "probe", + }, + }, + } + + probe_dir = args.artifact_dir / "src" + probe_dir.mkdir(parents=True, exist_ok=True) + probe_path = probe_dir / "webhook-probe.yaml" + with open(probe_path, "w", encoding="utf-8") as f: + yaml.safe_dump(probe_manifest, f, sort_keys=False) + + result = oc( + "apply", + "--dry-run=server", + "-f", + str(probe_path), + "-n", + args.namespace, + check=False, + ) + + if result.returncode == 0: + return "Webhook is ready (dry-run accepted)" + + return False if __name__ == "__main__": From 9392eea2f72eb6d64a11f55f267b96849f13e990 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 11:00:17 +0100 Subject: [PATCH 0064/1072] feat: Improve GH comment support Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/pr_args.py | 32 +++++++++-- projects/llm_d/tests/test_pr_args.py | 34 +++++++++++ projects/rhoai/library/pr_args.py | 75 +++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 projects/rhoai/library/pr_args.py diff --git a/projects/llm_d/orchestration/pr_args.py b/projects/llm_d/orchestration/pr_args.py index 6811a4726..7c11fd048 100644 --- a/projects/llm_d/orchestration/pr_args.py +++ b/projects/llm_d/orchestration/pr_args.py @@ -2,14 +2,22 @@ """ llm_d Project PR Arguments Parser -llm_d relies on the framework preset mechanism from `/test fournos llm_d PRESET`. -No llm_d-specific PR directives are currently required. +Parses llm_d-specific directives from PR trigger comments. + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 """ from __future__ import annotations +import logging from typing import Any +from projects.rhoai.library.pr_args import get_rhoai_directive_handlers + +logger = logging.getLogger(__name__) + def get_supported_llm_d_directives() -> dict[str, str]: """ @@ -31,5 +39,21 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st Returns: Tuple of (configuration overrides dict, list of parsed directive lines) """ - _ = comment_text - return {}, [] + directive_handlers = get_rhoai_directive_handlers() + config_overrides: dict[str, Any] = {} + parsed_directives: list[str] = [] + + for raw_line in comment_text.splitlines(): + line = raw_line.strip() + if not line: + continue + + for prefix, handler in directive_handlers.items(): + if line.startswith(prefix): + result = handler(line) + config_overrides.update(result) + parsed_directives.append(line) + logger.info("Parsed llm_d directive: %s", line) + break + + return config_overrides, parsed_directives diff --git a/projects/llm_d/tests/test_pr_args.py b/projects/llm_d/tests/test_pr_args.py index 11ae6a13d..9545a9e30 100644 --- a/projects/llm_d/tests/test_pr_args.py +++ b/projects/llm_d/tests/test_pr_args.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from projects.llm_d.orchestration import pr_args @@ -22,3 +24,35 @@ def test_parse_project_directives_ignores_other_projects() -> None: assert overrides == {} assert directives == [] + + +def test_rhoai_rc_image_sets_custom_catalog_config() -> None: + image = "quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310" + overrides, directives = pr_args.parse_project_directives(f"/rhoai.rc-image {image}") + + assert overrides == { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + assert len(directives) == 1 + + +def test_rhoai_rc_image_without_image_raises() -> None: + with pytest.raises(ValueError, match="image cannot be empty"): + pr_args.parse_project_directives("/rhoai.rc-image") + + +def test_rhoai_rc_image_with_other_directives() -> None: + comment = """/test fournos llm_d smoke +/rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123""" + + overrides, directives = pr_args.parse_project_directives(comment) + + assert overrides["platform.rhoai.custom_catalog.enabled"] is True + assert ( + overrides["platform.rhoai.custom_catalog.image"] + == "quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + assert overrides["platform.operators.rhods-operator.channel"] == "beta" + assert len(directives) == 1 diff --git a/projects/rhoai/library/pr_args.py b/projects/rhoai/library/pr_args.py new file mode 100644 index 000000000..6d10dccd6 --- /dev/null +++ b/projects/rhoai/library/pr_args.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +RHOAI PR directive handlers. + +Provides directive handlers for RHOAI-specific PR trigger directives. +These handlers are consumed by project-level pr_args modules (e.g., llm_d). + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 +""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def handle_rhoai_rc_image_directive(line: str) -> dict[str, Any]: + """ + Handle /rhoai.rc-image directive for setting RHOAI RC catalog image. + + Format: /rhoai.rc-image IMAGE + + Args: + line: The directive line + + Returns: + Dictionary with custom catalog and operator channel configuration + + Raises: + ValueError: If image is empty + """ + image = line.removeprefix("/rhoai.rc-image").strip() + + if not image: + raise ValueError( + "Invalid /rhoai.rc-image directive: image cannot be empty. " + "Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + + return { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + + +def get_rhoai_directive_handlers() -> dict[str, callable]: + """ + Get a mapping of RHOAI directive prefixes to their handler functions. + + Returns: + Dictionary mapping directive prefixes to handler functions + """ + return { + "/rhoai.rc-image": handle_rhoai_rc_image_directive, + } + + +def get_supported_rhoai_directives() -> dict[str, str]: + """ + Get a dictionary of supported RHOAI directives and their descriptions. + + Returns: + Dictionary mapping directive names to detailed descriptions + """ + return { + "/rhoai.rc-image": """Set RHOAI RC catalog image for testing release candidates. + Format: /rhoai.rc-image IMAGE + Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 + Effect: Enables custom catalog with the given image and sets operator channel to beta.""", + } From 1d335e895a9eae8e5552f2c6945bc428ef70d2bc Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 15 Jul 2026 11:13:07 +0100 Subject: [PATCH 0065/1072] fix: Cleanup ztunnels CRD Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/cleanup_phase.py | 28 ++++++++++++++++++- .../orchestration/config.d/platform.yaml | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index 0bf1e0e86..a6cfafd62 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -5,7 +5,7 @@ from projects.cluster.toolbox.cleanup_operators import ( main as cleanup_operators_command, ) -from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_resource_exists from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) @@ -73,6 +73,8 @@ def cleanup_operators(*, dry_run: bool = False) -> None: dry_run=dry_run, ) + _cleanup_stale_crds(platform, dry_run=dry_run) + def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[dict]: """Identify which operators should be cleaned up based on configuration.""" @@ -112,3 +114,27 @@ def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[ logger.info(f"Identified {len(operators_to_delete)} operators for deletion") return operators_to_delete + + +def _cleanup_stale_crds(platform: dict, *, dry_run: bool = False) -> None: + """Delete CRDs listed under cleanup_crds in operator specs. + + Some operators (e.g. rhcl-operator) leave CRDs with stored API versions + that block reinstallation on the next run. + """ + operators = platform.get("operators", {}) + if not isinstance(operators, dict): + return + + for package_name, operator_spec in operators.items(): + if not isinstance(operator_spec, dict): + continue + for crd in operator_spec.get("cleanup_crds", []): + if not oc_resource_exists("crd", crd): + logger.info("CRD %s not present, skipping", crd) + continue + if dry_run: + logger.info("[DRY RUN] Would delete CRD %s (from %s)", crd, package_name) + continue + logger.info("Deleting stale CRD %s (from %s)", crd, package_name) + oc("delete", "crd", crd, "--ignore-not-found=true", "--timeout=60s", check=False) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index bf8ec298b..b9defea06 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -41,6 +41,8 @@ operators: namespace: openshift-operators channel: stable source: redhat-operators + cleanup_crds: + - ztunnels.sailoperator.io rhods-operator: display_name: Red Hat OpenShift AI namespace: redhat-ods-operator From d8564dba5e969cd06f20b0ad5e409518497f4fe8 Mon Sep 17 00:00:00 2001 From: Harshith u <44830597+Harshith-umesh@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:02:46 -0400 Subject: [PATCH 0066/1072] Add Harshith-umesh to OWNERS and reviewers list --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 3fc847a58..ca8ee10bb 100644 --- a/OWNERS +++ b/OWNERS @@ -6,6 +6,7 @@ approvers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh reviewers: - ashishkamra - albertoperdomo2 @@ -13,3 +14,4 @@ reviewers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh From ced07f86e2ff5cd59e7b60a9ea17c3dff5ee047c Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 09:40:38 +0100 Subject: [PATCH 0067/1072] feat: Isolate benchmark workloads with per-deployment lifecycle Signed-off-by: Alberto Perdomo --- projects/llm_d/docs/benchmark-isolation.md | 109 +++++++++++++ projects/llm_d/orchestration/ci.py | 9 +- projects/llm_d/orchestration/cleanup_phase.py | 6 +- .../llm_d/orchestration/runtime_config.py | 67 +++++--- projects/llm_d/orchestration/test_phase.py | 69 ++++---- projects/llm_d/tests/test_profiles.py | 152 +++++++++++++++++- 6 files changed, 337 insertions(+), 75 deletions(-) create mode 100644 projects/llm_d/docs/benchmark-isolation.md diff --git a/projects/llm_d/docs/benchmark-isolation.md b/projects/llm_d/docs/benchmark-isolation.md new file mode 100644 index 000000000..c6a357c71 --- /dev/null +++ b/projects/llm_d/docs/benchmark-isolation.md @@ -0,0 +1,109 @@ +# Benchmark Isolation: Per-Workload Deployment + +## Why each benchmark needs a fresh deployment + +The llm_d test harness deploys an LLMInferenceService (LLMISVC) for each +benchmark workload rather than sharing a single deployment across workloads. +Three factors drive this design: + +### 1. `--max-model-len` affects performance + +The vLLM serving engine's `--max-model-len` parameter controls the maximum +sequence length and directly impacts memory allocation, scheduling behavior, +and throughput. Different workloads have different token distributions: + +- `short` uses `prompt_tokens=256, output_tokens=128` +- `heavy-heterogeneous` uses prompts up to 30,000 tokens + +Running `heavy-heterogeneous` against a server configured with +`--max-model-len=4096` would truncate or reject requests, while running +`short` against `--max-model-len=32768` wastes GPU memory on unused KV-cache +blocks. Workloads can provide `deployment_overrides` to set the right +`--max-model-len` for their token distribution. + +### 2. KV-cache pollution across workloads + +When multiple benchmarks share a deployment, residual KV-cache entries from +the first workload can affect the performance measurements of subsequent +workloads. The current mitigation relies on guidellm's prefixed prompts to +generate unique cache keys per benchmark. This works for standard load +generation but is not a guarantee of isolation. + +### 3. Future trace-replay workloads + +Trace-replay workloads (replaying production request traces) will require +starting the vLLM server with `VLLM_DEV_MODE=1` so that the KV-cache can +be explicitly cleared between runs. This is a deployment-time environment +variable, not a runtime toggle, reinforcing the need for per-workload +deployment isolation. + +## Architecture: 3D RunSpec matrix + +The test matrix is a Cartesian product of three dimensions: + +``` +model_name x deployment_profile x benchmark_key +``` + +Each combination (a `RunSpec`) gets: +1. Its own Kubernetes namespace +2. A fresh LLMISVC deployment with the profile's vLLM args +3. A smoke test to verify the deployment is healthy +4. A single benchmark run +5. Full teardown (LLMISVC, benchmark job, PVC, namespace resources) + +### Example + +``` +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.deployment_profile: [approximate-prefix-cache, distributed-default] +/var runtime.benchmark_key: [short, multi-turn] +``` + +Produces 1 x 2 x 2 = 4 isolated test runs, each with its own namespace: + +| RunSpec | Model | Profile | Benchmark | +|---------|-------|---------|-----------| +| 1 | Qwen3-0.6B | approximate-prefix-cache | short | +| 2 | Qwen3-0.6B | approximate-prefix-cache | multi-turn | +| 3 | Qwen3-0.6B | distributed-default | short | +| 4 | Qwen3-0.6B | distributed-default | multi-turn | + +## Backward compatibility + +| Scenario | `benchmark_key` value | Matrix | Behavior | +|----------|-----------------------|--------|----------| +| Smoke-only | `null` | M x P | No benchmark, just smoke test | +| Single benchmark | `"short"` | M x P x 1 | One benchmark per deploy (same as before) | +| Multiple benchmarks | `["a", "b"]` | M x P x 2 | Each gets its own deployment | + +When `benchmark_key` is null or a single scalar, the matrix degrades to the +previous 2D behavior with no overhead. + +## Per-workload deployment overrides + +Workloads can override deployment profile settings via `deployment_overrides` +in `config.d/workloads.yaml`. This dict is deep-merged on top of the resolved +deployment profile when building the LLMISVC manifest. + +```yaml +benchmarks: + heavy-heterogeneous: + args: + backend_type: openai_http + rate_type: concurrent + rate: [300, 200, 100, 50, 1] + data: prompt_tokens=8000,... + deployment_overrides: + vllm_args: + - --max-model-len=32768 + + trace-replay: + args: + backend_type: openai_http + ... + deployment_overrides: + env: + - name: VLLM_DEV_MODE + value: "1" +``` diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index b3748f679..6dc049806 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -107,10 +107,17 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" + from projects.llm_d.orchestration import runtime_config + # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) - return test_toolbox_run() + exit_code = 0 + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + with env.NextArtifactDir(run_spec.artifact_dirname): + exit_code = max(exit_code, test_toolbox_run()) + return exit_code @main.command() diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index a6cfafd62..1fac3327c 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -30,11 +30,7 @@ def cleanup_namespace(*, namespace: str | None = None) -> None: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() inference_service_name = platform["inference_service"]["name"] - benchmark_job_names = runtime_config.get_benchmark_job_names() - # Multiple benchmarks may share a job_name (defaults to "guidellm-benchmark"); - # the broad labeled sweep below deletes every forge-labeled job/pod/pvc - # regardless, so a single representative name is enough for the named pass. - benchmark_name = benchmark_job_names[0] if benchmark_job_names else None + benchmark_name = runtime_config.get_benchmark_job_name() if not oc_resource_exists("namespace", namespace): return diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index bc1ce1adf..dabbe22af 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -28,6 +28,8 @@ class RunSpec: model_slug: str deployment_profile_name: str deployment_profile_slug: str + benchmark_key: str | None + benchmark_slug: str | None namespace: str artifact_dirname: str namespace_is_managed: bool @@ -99,14 +101,14 @@ def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: return [value] if value else [] -def _deep_merge(base: Any, override: Any) -> Any: +def deep_merge(base: Any, override: Any) -> Any: if not isinstance(base, dict) or not isinstance(override, dict): return copy.deepcopy(override) merged = copy.deepcopy(base) for key, value in override.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = _deep_merge(merged[key], value) + merged[key] = deep_merge(merged[key], value) else: merged[key] = copy.deepcopy(value) return merged @@ -235,7 +237,7 @@ def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: benchmark_args = benchmark.get("args", {}) workload_args = workload_defaults.get("args", {}) if workload_args: - benchmark["args"] = _deep_merge(workload_args, benchmark_args) + benchmark["args"] = deep_merge(workload_args, benchmark_args) return benchmark @@ -253,22 +255,20 @@ def get_benchmark_config() -> dict[str, Any] | None: return _resolve_benchmark_config(benchmark_keys[0]) -def get_benchmark_configs() -> list[tuple[str, dict[str, Any]]]: - return [ - (benchmark_key, _resolve_benchmark_config(benchmark_key)) - for benchmark_key in get_benchmark_keys() - ] +def get_benchmark_deployment_overrides() -> dict[str, Any]: + """Return deployment overrides from the active benchmark config, if any.""" + benchmark = get_benchmark_config() + if benchmark is None: + return {} + return benchmark.get("deployment_overrides", {}) -def get_benchmark_job_names() -> list[str]: - """Distinct k8s benchmark job names for the active run, in order. - - A single benchmark_key collapses to one name; multiple benchmarks that - share a default job_name dedupe to one entry. Empty when benchmarking - is disabled, so cleanup paths can treat it uniformly. - """ - all_names = [benchmark.get("job_name") for _, benchmark in get_benchmark_configs()] - return list(dict.fromkeys(name for name in all_names if name)) +def get_benchmark_job_name() -> str | None: + """The k8s benchmark job name for the active run, or None when benchmarking is disabled.""" + benchmark = get_benchmark_config() + if benchmark is None: + return None + return benchmark.get("job_name") def get_deployment_profile_name() -> str: @@ -291,10 +291,10 @@ def get_deployment_profile() -> dict[str, Any]: defaults = copy.deepcopy(deployment_config.get("defaults", {})) scheduler_manifest = profile.pop("scheduler_manifest", None) - resolved_profile = _deep_merge(defaults, profile) + resolved_profile = deep_merge(defaults, profile) if scheduler_manifest: scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - resolved_profile = _deep_merge(resolved_profile, scheduler_data) + resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile @@ -312,6 +312,10 @@ def get_run_specs() -> list[RunSpec]: _get_runtime_value("deployment_profile"), "runtime.deployment_profile", ) + benchmark_keys = _normalize_string_or_list( + _get_runtime_value("benchmark_key"), + "runtime.benchmark_key", + ) if not model_names: raise ValueError("runtime.model_name must be set to a model name or list of model names") @@ -320,25 +324,37 @@ def get_run_specs() -> list[RunSpec]: "runtime.deployment_profile must be set to a deployment profile or list of profiles" ) - combinations = list(product(model_names, profile_names)) + if benchmark_keys: + benchmark_entries = [ + (key, slugify_identifier(key, max_length=24)) for key in benchmark_keys + ] + else: + benchmark_entries = [(None, None)] + + combinations = list(product(model_names, profile_names, benchmark_entries)) base_namespace = _derive_base_namespace() namespace_max_length = get_platform_config()["cluster"]["namespace"]["max_length"] # Compute base managed state once (ignores per-spec overrides) namespace_is_managed = get_namespace_is_managed() run_specs: list[RunSpec] = [] - for model_name, profile_name in combinations: + for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) if len(combinations) == 1: namespace = base_namespace artifact_dirname = "llmd_run" else: + ns_parts = [base_namespace, model_slug, profile_slug] + dir_parts = ["llmd_run", profile_slug, model_slug] + if bench_slug is not None: + ns_parts.append(bench_slug) + dir_parts.append(bench_slug) namespace = truncate_k8s_name( - f"{base_namespace}-{model_slug}-{profile_slug}", + "-".join(ns_parts), max_length=namespace_max_length, ) - artifact_dirname = f"llmd_run_{profile_slug}_{model_slug}" + artifact_dirname = "_".join(dir_parts) run_specs.append( RunSpec( @@ -346,6 +362,8 @@ def get_run_specs() -> list[RunSpec]: model_slug=model_slug, deployment_profile_name=profile_name, deployment_profile_slug=profile_slug, + benchmark_key=bench_key, + benchmark_slug=bench_slug, namespace=namespace, artifact_dirname=artifact_dirname, namespace_is_managed=namespace_is_managed, @@ -364,12 +382,14 @@ def activate_run_spec(run_spec: RunSpec): "model_name": _get_runtime_value("model_name"), "deployment_profile": _get_runtime_value("deployment_profile"), "namespace_override": _get_runtime_value("namespace_override"), + "benchmark_key": _get_runtime_value("benchmark_key"), } prev_managed_override = _namespace_is_managed_override config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) config.project.set_config("runtime.namespace_override", run_spec.namespace) + config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) _namespace_is_managed_override = run_spec.namespace_is_managed try: yield @@ -377,6 +397,7 @@ def activate_run_spec(run_spec: RunSpec): config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) config.project.set_config("runtime.namespace_override", saved["namespace_override"]) + config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) _namespace_is_managed_override = prev_managed_override diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2ef047ef2..0af65e07e 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -40,11 +40,8 @@ def create_test_labels() -> None: "deployment_profile": deployment_profile, } - # Add guidellm configuration information if benchmark_keys: - labels["guidellm_loadshape"] = ( - benchmark_keys[0] if len(benchmark_keys) == 1 else benchmark_keys - ) + labels["guidellm_loadshape"] = benchmark_keys[0] write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) @@ -222,6 +219,10 @@ def _build_inference_service_manifest() -> Path: deployment_profile = runtime_config.get_deployment_profile() model_cache = runtime_config.get_model_cache_config() + benchmark_overrides = runtime_config.get_benchmark_deployment_overrides() + if benchmark_overrides: + deployment_profile = runtime_config.deep_merge(deployment_profile, benchmark_overrides) + # Build the InferenceService manifest manifest = render_inference_service_from_parts( config_dir=config_dir, @@ -267,30 +268,29 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: def run_guidellm_benchmark(*, endpoint_url: str) -> None: - # Load config where it's consumed from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() - benchmark_configs = runtime_config.get_benchmark_configs() - - if not benchmark_configs: - return # Skip if benchmark is disabled - - for benchmark_key, benchmark in benchmark_configs: - guidellm_args = build_guidellm_args(benchmark) - if not any(arg.startswith("--processor=") for arg in guidellm_args): - guidellm_args.append(f"--processor={runtime_config.get_model_name()}") - artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" - with env.NextArtifactDir(artifact_name): - run_guidellm_benchmark_command.run( - endpoint_url=endpoint_url, - name=benchmark.get("job_name"), - namespace=namespace, - image=benchmark.get("image"), - timeout=benchmark.get("timeout_seconds"), - pvc_size=benchmark.get("pvc_size"), - guidellm_args=guidellm_args, - ) + benchmark = runtime_config.get_benchmark_config() + + if benchmark is None: + return + + benchmark_key = runtime_config.get_benchmark_keys()[0] + guidellm_args = build_guidellm_args(benchmark) + if not any(arg.startswith("--processor=") for arg in guidellm_args): + guidellm_args.append(f"--processor={runtime_config.get_model_name()}") + artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" + with env.NextArtifactDir(artifact_name): + run_guidellm_benchmark_command.run( + endpoint_url=endpoint_url, + name=benchmark.get("job_name"), + namespace=namespace, + image=benchmark.get("image"), + timeout=benchmark.get("timeout_seconds"), + pvc_size=benchmark.get("pvc_size"), + guidellm_args=guidellm_args, + ) def capture_inference_service_state() -> None: @@ -318,7 +318,6 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" - # Load config where it's consumed from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() @@ -326,18 +325,12 @@ def cleanup_test_resources() -> None: inference_service = platform["inference_service"] smoke = platform["smoke"] - # Narrow cleanup (no broad sweep): cover every benchmark job name. Benchmarks - # typically share a single job_name, so this dedupes to one call; when - # benchmarking is disabled, run once with None so the IS/smoke cleanup still - # runs. oc delete is idempotent across iterations. - benchmark_job_names = runtime_config.get_benchmark_job_names() or [None] - for benchmark_job_name in benchmark_job_names: - cleanup_test_resources_command.run( - namespace=namespace, - inference_service_name=inference_service["name"], - smoke_pod_name=smoke["pod_name"], - benchmark_job_name=benchmark_job_name, - ) + cleanup_test_resources_command.run( + namespace=namespace, + inference_service_name=inference_service["name"], + smoke_pod_name=smoke["pod_name"], + benchmark_job_name=runtime_config.get_benchmark_job_name(), + ) def capture_namespace_events_after_test( diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 6b9c948d4..c4df70be7 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -165,10 +165,31 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non "multi-turn", ] - benchmark_configs = dict(runtime_config.get_benchmark_configs()) - assert benchmark_configs["concurrent-1k-1k"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["heavy-heterogeneous"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["multi-turn"]["args"]["request_type"] == "text_completions" + run_specs = runtime_config.get_run_specs() + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + benchmark = runtime_config.get_benchmark_config() + assert benchmark["args"]["request_type"] == "text_completions" + + +def test_release_preset_produces_3_run_specs() -> None: + _init_project_config() + + core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 3 + assert [spec.benchmark_key for spec in run_specs] == [ + "concurrent-1k-1k", + "heavy-heterogeneous", + "multi-turn", + ] + assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) + assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) + + namespaces = [spec.namespace for spec in run_specs] + assert len(namespaces) == len(set(namespaces)) def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: @@ -544,6 +565,116 @@ def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: ] assert run_specs[0].model_slug == "openai-gpt-oss-120b" assert run_specs[2].model_slug == "qwen-qwen3-0-6b" + # With a single benchmark_key (or null), benchmark fields reflect the scalar + assert all(spec.benchmark_key is not None or spec.benchmark_slug is None for spec in run_specs) + + +def test_3d_matrix_with_multiple_benchmark_keys() -> None: + _init_project_config() + + core_config.project.set_config( + "runtime.model_name", + "[openai/gpt-oss-120b, Qwen/Qwen3-0.6B]", + ) + core_config.project.set_config( + "runtime.deployment_profile", + "[distributed-default, precise-prefix-cache]", + ) + core_config.project.set_config( + "runtime.benchmark_key", + "[concurrent-1k-1k, multi-turn]", + ) + + run_specs = runtime_config.get_run_specs() + + # 2 models x 2 profiles x 2 benchmarks = 8 specs + assert len(run_specs) == 8 + assert all(spec.benchmark_key is not None for spec in run_specs) + assert all(spec.benchmark_slug is not None for spec in run_specs) + + assert run_specs[0].model_name == "openai/gpt-oss-120b" + assert run_specs[0].deployment_profile_name == "distributed-default" + assert run_specs[0].benchmark_key == "concurrent-1k-1k" + + # Verify all namespaces are unique + namespaces = [spec.namespace for spec in run_specs] + assert len(namespaces) == len(set(namespaces)) + + # Verify all artifact dirnames are unique + dirnames = [spec.artifact_dirname for spec in run_specs] + assert len(dirnames) == len(set(dirnames)) + + +def test_null_benchmark_key_produces_smoke_only_specs() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key is None + assert run_specs[0].benchmark_slug is None + + +def test_single_benchmark_key_backward_compatible() -> None: + _init_project_config() + + core_config.project.apply_preset("smoke") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key == "short" + assert run_specs[0].benchmark_slug == "short" + assert run_specs[0].artifact_dirname == "llmd_run" + + +def test_activate_run_spec_sets_benchmark_key() -> None: + _init_project_config() + + core_config.project.set_config( + "runtime.benchmark_key", + "[concurrent-1k-1k, multi-turn]", + ) + + run_specs = runtime_config.get_run_specs() + + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + keys = runtime_config.get_benchmark_keys() + assert keys == [run_spec.benchmark_key] + assert runtime_config.get_benchmark_config() is not None + + +def test_benchmark_deployment_overrides() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + core_config.project.config["workloads"]["benchmarks"]["concurrent-1k-1k"][ + "deployment_overrides" + ] = {"vllm_args": ["--max-model-len=8192"]} + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {"vllm_args": ["--max-model-len=8192"]} + + +def test_benchmark_deployment_overrides_empty_when_not_set() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} + + +def test_benchmark_deployment_overrides_empty_when_no_benchmark() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} def test_runtime_rejects_legacy_model_key_path() -> None: @@ -665,12 +796,14 @@ def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> N assert "scheduler" not in manifest["spec"]["router"] -def test_benchmark_job_names_collapse_shared_default() -> None: +def test_benchmark_job_name_from_activated_spec() -> None: _init_project_config() core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") - # Three benchmark_keys, all sharing the workload default job_name -> one entry. - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] + + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" def test_smoke_preset_benchmark_behavior() -> None: @@ -679,7 +812,10 @@ def test_smoke_preset_benchmark_behavior() -> None: core_config.project.apply_preset("smoke") # The smoke preset enables the short benchmark assert runtime_config.get_benchmark_keys() == ["short"] - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] + + run_spec = runtime_config.get_run_specs()[0] + with runtime_config.activate_run_spec(run_spec): + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" def test_run_spec_preserves_namespace_managed_flag() -> None: From b8b04e35c9f490444c28aa8715da3ca73fcb2350 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 10 Jul 2026 14:43:06 +0200 Subject: [PATCH 0068/1072] fournos: gitops: base/workflows/task-forge-step: update for clusterless jobs --- .../base/workflows/task-forge-step.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index dcb29b30c..90f746be6 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,6 +13,8 @@ spec: type: string - name: kubeconfig-secret type: string + default: "fournos-clusterless-placeholder" + description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -31,6 +33,8 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig + - name: KUBECONFIG_SECRET_PARAM + value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -73,7 +77,21 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - export KUBECONFIG=$TARGET_KUBECONFIG + # Check if we have a kubeconfig for target cluster + if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then + echo "Setting up target cluster kubeconfig..." + export KUBECONFIG=$TARGET_KUBECONFIG + export CLUSTERLESS_MODE="false" + + else + if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then + echo "Running in clusterless mode - no target cluster kubeconfig" + else + echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" + echo "Running in clusterless mode as fallback" + fi + export CLUSTERLESS_MODE="true" + fi # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -126,3 +144,4 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) + optional: true From b4b4437e7c5d05d383ff4e6e5d6d884ee09f88eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 11:45:01 +0200 Subject: [PATCH 0069/1072] [skeleton] orchestration: ci: add the launcher stub --- projects/skeleton/orchestration/ci.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/skeleton/orchestration/ci.py b/projects/skeleton/orchestration/ci.py index 6dfef28ac..d8efa8e5c 100755 --- a/projects/skeleton/orchestration/ci.py +++ b/projects/skeleton/orchestration/ci.py @@ -102,6 +102,17 @@ def preflight(ctx) -> int: return 0 +@main.command() +@click.pass_context +@ci_lib.safe_ci_entrypoint +def launcher(ctx) -> int: + """Launcher job - Deploys fournos jobs.""" + + logger.warning("Nothing so far for the launcher job") + + return 0 + + main.add_command(caliper_export_entrypoint) main.add_command(caliper_replot_entrypoint) From 889f94efea9208e163014d1cb776eb321187d87b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:33:43 +0200 Subject: [PATCH 0070/1072] fournos: gitops: base/workflows/pipeline-launcher: new pipeline to invoke a clusterless launcher --- .../base/workflows/pipeline-launcher.yaml | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 fournos/gitops/base/workflows/pipeline-launcher.yaml diff --git a/fournos/gitops/base/workflows/pipeline-launcher.yaml b/fournos/gitops/base/workflows/pipeline-launcher.yaml new file mode 100644 index 000000000..bca8f0fe8 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-launcher.yaml @@ -0,0 +1,77 @@ +# Single-step pipeline: run only (no prepare, no cleanup). +# Use on pre-configured clusters where setup/teardown is handled externally. +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: forge-launcher + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-preflight + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: preflight + - name: job-step-name + value: 00__preflight + + - name: 01-launcher + runAfter: [00-preflight] + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: launcher + - name: job-step-name + value: 01__launcher + + finally: + - name: 02-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 02__export-artifacts From 7bf1e0e39f3bfc116e493ae5159297a1e4b16a00 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:34:52 +0200 Subject: [PATCH 0071/1072] [fournos_launcher] orchestration: pr_args: add support for the /clusterless directive --- .../fournos_launcher/orchestration/pr_args.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index b0003acea..4d7a3300b 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -40,6 +40,11 @@ def get_supported_fournos_directives() -> dict[str, str]: Format: /exclusive true|false Example: /exclusive false Effect: Sets fournos.job.exclusive in configuration (default: true).""", + "/clusterless": """Enable clusterless mode for FOURNOS job execution. + Format: /clusterless + Example: /clusterless + Effect: Sets fournos.job.exclusive=false and fournos.job.clusterless=true. + Note: Clusterless and exclusive modes are mutually exclusive.""", "/pipeline": """Set FOURNOS pipeline name for job execution. Format: /pipeline pipeline_name Example: /pipeline llm-load-test @@ -115,6 +120,21 @@ def handle_exclusive_directive(line: str) -> dict[str, str]: return {"fournos.job.exclusive": exclusive_value == "true"} +def handle_clusterless_directive(line: str) -> dict[str, str]: + """ + Handle /clusterless directive for enabling clusterless mode. + + Format: /clusterless + + Args: + line: The directive line + + Returns: + Dictionary with clusterless configuration (sets exclusive=false, clusterless=true) + """ + return {"fournos.job.exclusive": False, "fournos.job.clusterless": True} + + def handle_pipeline_directive(line: str) -> dict[str, str]: """ Handle /pipeline directive for setting pipeline name. @@ -291,6 +311,7 @@ def get_fournos_directive_handlers() -> dict[str, callable]: return { "/cluster": handle_cluster_directive, "/exclusive": handle_exclusive_directive, + "/clusterless": handle_clusterless_directive, "/pipeline": handle_pipeline_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, @@ -324,6 +345,14 @@ def parse_fournos_directives( required_directives=None, # No required directives for FOURNOS ) + # Validate directive conflicts + if config_overrides.get("fournos.job.clusterless") and config_overrides.get( + "fournos.job.exclusive" + ): + raise ValueError( + "Conflicting directives: /clusterless and /exclusive true cannot both be used" + ) + # Log successful parses at info level for FOURNOS for directive in parsed_directives: logger.info(f"Parsed FOURNOS directive: {directive}") From d74d9f12f16327158a0bed48550f217052d00245 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:35:51 +0200 Subject: [PATCH 0072/1072] [fournos_launcher] orchestration: config: add clusterless field --- projects/fournos_launcher/orchestration/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 4ff135edc..5be5ca092 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -25,6 +25,7 @@ fournos: pipeline_name: forge-test-only ci_label: null # will be generated from the CI job UID exclusive: true + clusterless: null env: OPENSHIFT_CI_job: [JOB_TYPE, JOB_NAME, JOB_SPEC, OPENSHIFT_CI, JOB_NAME_SAFE, BUILD_ID] OPENSHIFT_CI_git_pr: [PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_REF, REPO_NAME, REPO_OWNER, PULL_BASE_SHA, JOB_NAME, PULL_TITLE, PULL_REFS, PULL_HEAD_REF] From f5cb9c5e5ae9790c004348b5261be935f2fe7b4f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:41:57 +0200 Subject: [PATCH 0073/1072] [fournos_launcher] orchestration: pr_args: add the /fournos directive --- .../fournos_launcher/orchestration/pr_args.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index 4d7a3300b..270d83dd8 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -45,6 +45,11 @@ def get_supported_fournos_directives() -> dict[str, str]: Example: /clusterless Effect: Sets fournos.job.exclusive=false and fournos.job.clusterless=true. Note: Clusterless and exclusive modes are mutually exclusive.""", + "/fournos": """Set FOURNOS namespace environment. + Format: /fournos environment + Example: /fournos wip + /fournos staging + Effect: Sets fournos.namespace to psap-automation-{environment}.""", "/pipeline": """Set FOURNOS pipeline name for job execution. Format: /pipeline pipeline_name Example: /pipeline llm-load-test @@ -135,6 +140,38 @@ def handle_clusterless_directive(line: str) -> dict[str, str]: return {"fournos.job.exclusive": False, "fournos.job.clusterless": True} +def handle_fournos_directive(line: str) -> dict[str, str]: + """ + Handle /fournos directive for setting FOURNOS namespace environment. + + Format: /fournos environment + + Args: + line: The directive line + + Returns: + Dictionary with namespace configuration + + Raises: + ValueError: If environment is not wip or staging + """ + FOURNOS_NAMESPACE_BASE = "psap-automation" + VALID_ENVIRONMENTS = {"wip", "staging"} + + environment = line.removeprefix("/fournos ").strip() + + if not environment: + raise ValueError(f"Invalid /fournos directive: environment cannot be empty in '{line}'") + + if environment not in VALID_ENVIRONMENTS: + raise ValueError( + f"Invalid /fournos directive: environment must be one of {VALID_ENVIRONMENTS}, got '{environment}' in '{line}'" + ) + + namespace = f"{FOURNOS_NAMESPACE_BASE}-{environment}" + return {"fournos.namespace": namespace} + + def handle_pipeline_directive(line: str) -> dict[str, str]: """ Handle /pipeline directive for setting pipeline name. @@ -312,6 +349,7 @@ def get_fournos_directive_handlers() -> dict[str, callable]: "/cluster": handle_cluster_directive, "/exclusive": handle_exclusive_directive, "/clusterless": handle_clusterless_directive, + "/fournos": handle_fournos_directive, "/pipeline": handle_pipeline_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, From f2662b1263d0c5c83d5f07301ee77a12ae4f772e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 12:51:38 +0200 Subject: [PATCH 0074/1072] [fournos_launcher] orchestration: submit: /cluster not mandatory if clusterless --- projects/fournos_launcher/orchestration/submit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index dbbcf5944..6c99a9b84 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -258,9 +258,11 @@ def submit_job(): # Validate required configuration before job submission cluster_name = config.project.get_config("cluster.name") - if not cluster_name: + clusterless_mode = config.project.get_config("fournos.job.clusterless") + + if not cluster_name and not clusterless_mode: raise ValueError( - "/cluster (cluster.name) must be set - cannot submit job without target cluster" + "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" ) # Get GPU hardware configuration From 32345a81ffaeed336804c9a4e61dcbdad2a720ed Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 13:53:57 +0200 Subject: [PATCH 0075/1072] [fournos_launcher] orchestration: more logs and checks for clusterless --- .../fournos_launcher/orchestration/cli.py | 22 ++++++++++++++++--- .../fournos_launcher/orchestration/submit.py | 8 +++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/projects/fournos_launcher/orchestration/cli.py b/projects/fournos_launcher/orchestration/cli.py index 4acd19eea..ff3eda17d 100755 --- a/projects/fournos_launcher/orchestration/cli.py +++ b/projects/fournos_launcher/orchestration/cli.py @@ -53,13 +53,29 @@ def submit(ctx, cluster, project, args, namespace, override, commit): logger.warning(f"Ignoring invalid override format: {override_str} (expected key=value)") # Override config values if provided + clusterless_mode = config.project.get_config("fournos.job.clusterless") + if cluster: config.project.set_config("cluster.name", cluster) else: cluster = config.project.get_config("cluster.name") - if not cluster: - raise ValueError("--cluster or cluster.name is mandatory") - logger.info(f"Using cluster {cluster}") + + if not cluster and not clusterless_mode: + raise ValueError( + "--cluster or cluster.name is mandatory (unless clusterless mode is enabled)" + ) + + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: + raise ValueError( + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" + ) + + if cluster: + logger.info(f"Using cluster {cluster}") + else: + logger.info("Using clusterless mode") if project: config.project.set_config("ci_job.project", project) diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index 6c99a9b84..fbe5ac3b8 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -265,6 +265,13 @@ def submit_job(): "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" ) + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: + raise ValueError( + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" + ) + # Get GPU hardware configuration gpu_count = config.project.get_config("fournos.job.hardware.gpu_count") gpu_type = config.project.get_config("fournos.job.hardware.gpu_type") @@ -296,6 +303,7 @@ def submit_job(): "env": env_dict, "ci_label": config.project.get_config("fournos.job.ci_label"), "exclusive": config.project.get_config("fournos.job.exclusive"), + "clusterless": config.project.get_config("fournos.job.clusterless"), "gpu_count": gpu_count, "gpu_type": gpu_type, } From c51e0c5eea701b08ee3b10f2bb1eead18847fa62 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 13:58:25 +0200 Subject: [PATCH 0076/1072] [fournos_launcher] toolbox: submit_and_wait: add support for spec.clusterless --- .../toolbox/submit_and_wait/main.py | 19 ++++++++++++++----- .../submit_and_wait/templates/job.yaml.j2 | 4 ++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 88f544a80..40e4917b6 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -47,9 +47,9 @@ def __init__( @entrypoint def run( - cluster_name: str, project: str, *, + cluster_name: str | None = None, args: list = None, variables_overrides: dict = None, job_name: str = "", @@ -63,13 +63,14 @@ def run( exclusive: bool = True, gpu_count: int = None, gpu_type: str = None, + clusterless: bool = False, ): """ Submit a FOURNOS job and wait for completion Args: - cluster_name: Name of the target cluster for the FOURNOS job project: The project name to execute (e.g., 'llm_d', 'skeleton') + cluster_name: Name of the target cluster for the FOURNOS job (None when clusterless=True) args: List of arguments to pass to the project (default: empty list) variables_overrides: Dictionary of config variables to override (default: empty dict) job_name: Custom name for the FOURNOS job (auto-generated if empty) @@ -83,12 +84,13 @@ def run( exclusive: Whether the job should run exclusively on its nodes (default: True) gpu_count: Number of GPUs required for the job (default: None) gpu_type: Type of GPU required for the job (default: None) + clusterless: Whether to enable clusterless mode (makes cluster_name optional, forces exclusive=False) (default: False) Examples: # Called by entrypoint with config values: run( + "llm_d", cluster_name="my-cluster", - project="llm_d", args=["test", "--verbose"], variables_overrides={"model": "mistral", "replicas": 2}, namespace="my-fournos-jobs", @@ -101,6 +103,10 @@ def run( gpu_type="nvidia-tesla-v100", ) """ + # Enforce clusterless=True always uses exclusive=False to prevent conflicts + if clusterless: + exclusive = False + # Set defaults if args is None: args = [] @@ -123,8 +129,11 @@ def run( def validate_inputs(args, ctx): """Validate input parameters""" - if not args.cluster_name: - raise ValueError("cluster_name is required") + if not args.cluster_name and not args.clusterless: + raise ValueError("cluster_name is required unless clusterless mode is enabled") + + if args.clusterless and args.exclusive: + raise ValueError("Clusterless mode and exclusive mode cannot both be enabled") if not args.project: raise ValueError("project is required") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 index 89ab7c5f8..07a84ea9b 100644 --- a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 +++ b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 @@ -10,7 +10,11 @@ metadata: spec: owner: {{ args.owner }} displayName: {{ args.display_name }} + {% if args.clusterless -%} + clusterless: {{ args.clusterless }} + {%- else -%} cluster: {{ args.cluster_name }} + {%- endif %} exclusive: {{ args.exclusive }} pipeline: {{ args.pipeline_name }} {% if args.gpu_count and args.gpu_type -%} From b0bb1d5f6d38fbe074bc50e93e669f65c88144de Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 13 Jul 2026 14:06:33 +0200 Subject: [PATCH 0077/1072] [projects] Add Changelog files --- .../CHANGELOG-orchestration.md | 31 +++++++++++++++++++ .../fournos_launcher/CHANGELOG-toolbox.md | 19 ++++++++++++ 2 files changed, 50 insertions(+) diff --git a/projects/fournos_launcher/CHANGELOG-orchestration.md b/projects/fournos_launcher/CHANGELOG-orchestration.md index 1430e84d9..e83eb539e 100644 --- a/projects/fournos_launcher/CHANGELOG-orchestration.md +++ b/projects/fournos_launcher/CHANGELOG-orchestration.md @@ -1,5 +1,36 @@ # Fournos Launcher Orchestration Changelog +## 2026-07-13 - Clusterless Mode and New Directives + +### New PR Directives +- **Clusterless Directive**: Added `/clusterless` directive to enable clusterless mode execution + - **Effect**: Sets `fournos.job.exclusive=false` and `fournos.job.clusterless=true` + - **Validation**: Prevents conflicting usage with `/exclusive true` +- **Fournos Environment Directive**: Added `/fournos` directive for namespace targeting + - **Format**: `/fournos wip` or `/fournos staging` + - **Effect**: Sets `fournos.namespace` to `psap-automation-{environment}` + - **Validation**: Restricts to `wip` and `staging` environments only + +### Enhanced Validation +- **Mutual Exclusivity**: Added comprehensive validation to prevent `clusterless=true` and `exclusive=true` conflicts + - **PR Parser Level**: Catches directive conflicts during comment parsing + - **CLI Level**: Validates configuration before command execution + - **Submit Level**: Final validation before job submission +- **Optional Cluster**: Made `/cluster` directive optional when `/clusterless` is used + - **Smart Validation**: Requires cluster specification only when not in clusterless mode + - **Clear Error Messages**: Improved error messaging to explain clusterless option + +### Files Modified +- `pr_args.py` - Added `/clusterless` and `/fournos` directive handlers with validation +- `cli.py` - Enhanced cluster requirement logic and added conflict validation +- `submit.py` - Added clusterless/exclusive mutual exclusivity checks + +### Benefits +- **Flexible Execution**: Enables both cluster-targeted and clusterless job execution modes +- **Environment Targeting**: Simplified namespace selection for different deployment environments +- **Robust Validation**: Prevents configuration conflicts with clear, actionable error messages +- **Backward Compatibility**: Maintains existing directive functionality while adding new capabilities + ## 2026-06-30 - Notification System Integration ### Job Completion Notifications diff --git a/projects/fournos_launcher/CHANGELOG-toolbox.md b/projects/fournos_launcher/CHANGELOG-toolbox.md index e4b0ee2bb..f16dc0ea9 100644 --- a/projects/fournos_launcher/CHANGELOG-toolbox.md +++ b/projects/fournos_launcher/CHANGELOG-toolbox.md @@ -1,5 +1,24 @@ # Fournos Launcher Toolbox Changelog +## 2026-07-13 - Clusterless Mode Support + +### submit_and_wait + +#### Clusterless Execution Support +- **Clusterless Parameter**: Added `clusterless: bool = False` parameter to entrypoint function +- **Template Updates**: Enhanced job.yaml.j2 template to conditionally render `clusterless: true` or `cluster: ` fields +- **Validation Logic**: Added validation to ensure clusterless and exclusive modes are mutually exclusive +- **Optional Cluster**: Made `cluster_name` parameter optional when `clusterless=True` + +#### Files Modified +- `main.py` - Added clusterless parameter support and validation logic +- `templates/job.yaml.j2` - Updated FournosJob template with conditional cluster/clusterless field rendering + +#### Benefits +- Enables clusterless job execution for improved resource flexibility +- Maintains backward compatibility with existing cluster-based workflows +- Provides clear validation for conflicting configuration options + ## 2026-06-24 - Job Management Improvements ### submit_and_wait From fcddf87678fd456127f33f2ebdd34d42d3161ef5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 14 Jul 2026 09:05:47 +0200 Subject: [PATCH 0078/1072] fournos: gitops: base/workflows/pipeline-prepare-test: fix --- fournos/gitops/base/workflows/pipeline-prepare-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml index 0070f0b0b..40a5a0216 100644 --- a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml +++ b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml @@ -57,7 +57,7 @@ spec: value: 02__preflight - name: 03-test - runAfter: [01-preflight] + runAfter: [02-preflight] taskRef: name: forge-step kind: Task From db896543a065ae8f51e13135ed4162165f4b5b02 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 14 Jul 2026 09:07:58 +0200 Subject: [PATCH 0079/1072] fournos: gitops: base/workflows/kustomization: add missing pipelines and alpha order --- fournos/gitops/base/workflows/kustomization.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fournos/gitops/base/workflows/kustomization.yaml b/fournos/gitops/base/workflows/kustomization.yaml index d56669a87..2212ab4a0 100644 --- a/fournos/gitops/base/workflows/kustomization.yaml +++ b/fournos/gitops/base/workflows/kustomization.yaml @@ -5,9 +5,11 @@ resources: # - resolve-job.yaml - task-forge-step.yaml - pipeline-full.yaml - - pipeline-test-only.yaml + - pipeline-launcher.yaml + - pipeline-prepare-only.yaml + - pipeline-prepare-test.yaml - pipeline-replot.yaml - + - pipeline-test-only.yaml commonLabels: app.kubernetes.io/name: forge-on-fournos app.kubernetes.io/component: workflows From b8f75a9d0ac8c6308903b554319059c096dc2450 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 15:27:36 +0100 Subject: [PATCH 0080/1072] feat: Add timeout to best effort delete Signed-off-by: Alberto Perdomo --- projects/llm_d/toolbox/cleanup_test_resources/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 90e12d77a..c4f0a461f 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -220,7 +220,7 @@ def wait_for_workload_pods_deletion(args, ctx): def _best_effort_delete(description: str, *oc_args: str) -> None: """Best effort deletion of Kubernetes resources with timeout""" try: - oc(*oc_args, check=False) + oc(*oc_args, "--timeout=120s", check=False) except subprocess.TimeoutExpired: logger.warning("Timed out deleting %s: oc %s", description, " ".join(oc_args)) From 7bbcf33aebd2a07a7ac84473397a9c69bd40bc79 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 18:01:43 +0100 Subject: [PATCH 0081/1072] chore: Fix truncate_k8s_name Signed-off-by: Alberto Perdomo --- projects/core/dsl/utils/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/projects/core/dsl/utils/__init__.py b/projects/core/dsl/utils/__init__.py index 296b0cccd..80effeb96 100644 --- a/projects/core/dsl/utils/__init__.py +++ b/projects/core/dsl/utils/__init__.py @@ -39,5 +39,13 @@ def slugify_identifier(value: str, *, max_length: int = 63) -> str: def truncate_k8s_name(value: str, *, max_length: int = 63) -> str: - """Truncate a string to Kubernetes name length limits.""" - return value[:max_length].rstrip("-") + """Truncate a string to Kubernetes name length limits. + + When truncation occurs, a short hash suffix is appended to avoid + collisions between names that share a long common prefix. + """ + if len(value) <= max_length: + return value + hash_suffix = hashlib.sha256(value.encode()).hexdigest()[:6] + prefix = value[: max_length - len(hash_suffix) - 1].rstrip("-") + return f"{prefix}-{hash_suffix}" From 22cf89482fbe03c84e150427ff0e2ef367a5493e Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 16 Jul 2026 18:02:21 +0100 Subject: [PATCH 0082/1072] feat: Enhance delete best effort Signed-off-by: Alberto Perdomo --- projects/llm_d/docs/benchmark-isolation.md | 4 +++- projects/llm_d/toolbox/cleanup_test_resources/main.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/docs/benchmark-isolation.md b/projects/llm_d/docs/benchmark-isolation.md index c6a357c71..3fae5d2b0 100644 --- a/projects/llm_d/docs/benchmark-isolation.md +++ b/projects/llm_d/docs/benchmark-isolation.md @@ -84,7 +84,9 @@ previous 2D behavior with no overhead. Workloads can override deployment profile settings via `deployment_overrides` in `config.d/workloads.yaml`. This dict is deep-merged on top of the resolved -deployment profile when building the LLMISVC manifest. +deployment profile when building the LLMISVC manifest. Lists (like `vllm_args`) +are **replaced entirely**, not concatenated — this avoids duplicate or +conflicting flags when the override supplies a different set of arguments. ```yaml benchmarks: diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index c4f0a461f..9afa1fb88 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -218,9 +218,15 @@ def wait_for_workload_pods_deletion(args, ctx): def _best_effort_delete(description: str, *oc_args: str) -> None: - """Best effort deletion of Kubernetes resources with timeout""" + """Best effort deletion of Kubernetes resources with timeout.""" try: - oc(*oc_args, "--timeout=120s", check=False) + result = oc(*oc_args, "--timeout=300s", check=False) + if result.returncode != 0: + logger.warning( + "Failed to delete %s (rc=%d), capturing describe", description, result.returncode + ) + describe_args = tuple(a for a in oc_args if a != "delete") + oc("describe", *describe_args, check=False) except subprocess.TimeoutExpired: logger.warning("Timed out deleting %s: oc %s", description, " ".join(oc_args)) From cbff983d574ff22a0028fae4e3b94833b7546c31 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 10 Jul 2026 11:49:08 -0400 Subject: [PATCH 0083/1072] Port model-furnace features to forge/rhaiis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CSV dashboard export (csv_export.py): generates consolidated_dashboard.csv from PSAP JSON files matching model-furnace's exact column schema - S3 sync (s3_dashboard.py): download/append/re-upload consolidated CSV and upload profiler traces, using new psap-forge-dashboard-s3 vault - Regression detection (analyze.py): geometric mean comparison with configurable thresholds per metric, ported from model-furnace's compare_results.py - Slack webhook notifications (send.py): regression/improvement alerts via Incoming Webhook (psap-forge-rhaiis-slack vault), with Bot Token fallback - Profiler toolbox tasks: enable/disable gate, copy traces from pod, and verify_profiler_prereqs (webhook pod, TLS, ConfigMap checks) - Test phase wiring: profiler prereq check → gate toggle → benchmark → trace copy → PSAP payload → CSV export → S3 sync → regression analysis → Slack → profiler trace upload → MLflow metadata - Config: new vault definitions, profiler section, csv_dashboard and analyze sections in rhaiis config Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 246 +++++++++++- projects/core/notifications/send.py | 151 ++++++++ .../rhaiis/orchestration/config.d/rhaiis.yaml | 7 + .../orchestration/config.d/workloads.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 17 + .../rhaiis/orchestration/runtime_config.py | 4 + projects/rhaiis/orchestration/test_phase.py | 355 ++++++++++++++++-- projects/rhaiis/postprocess/csv_export.py | 184 +++++++++ projects/rhaiis/postprocess/s3_dashboard.py | 175 +++++++++ .../toolbox/copy_profiler_traces/main.py | 67 ++++ .../toolbox/enable_profiler_gate/main.py | 50 +++ .../toolbox/verify_profiler_prereqs/main.py | 127 +++++++ vaults/psap-forge-dashboard-s3.yaml | 16 + vaults/psap-forge-rhaiis-slack.yaml | 10 + 14 files changed, 1389 insertions(+), 24 deletions(-) create mode 100644 projects/rhaiis/postprocess/csv_export.py create mode 100644 projects/rhaiis/postprocess/s3_dashboard.py create mode 100644 projects/rhaiis/toolbox/copy_profiler_traces/main.py create mode 100644 projects/rhaiis/toolbox/enable_profiler_gate/main.py create mode 100644 projects/rhaiis/toolbox/verify_profiler_prereqs/main.py create mode 100644 vaults/psap-forge-dashboard-s3.yaml create mode 100644 vaults/psap-forge-rhaiis-slack.yaml diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 825762b62..f2ef0b4bd 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,14 +2,258 @@ from __future__ import annotations +import csv import json import logging +import tempfile import time +from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Optional + +import numpy as np logger = logging.getLogger(__name__) +PROFILE_MAP = { + (1000, 1000): "profile1", + (512, 2048): "profile2", + (2048, 128): "profile3", + (8000, 1000): "profile4", +} + +METRICS = { + "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, + "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, + "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, + "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, + "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, +} + + +@dataclass +class RegressionResult: + profile: str + metric_column: str + metric_label: str + current_value: float + baseline_value: float + pct_diff: float + is_regression: bool + is_improvement: bool = False + + +def geometric_mean(values) -> Optional[float]: + positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + if len(positive) == 0: + return None + return float(np.exp(np.mean(np.log(positive)))) + + +def compare_runs( + current_csv_path: str, + consolidated_csv_path: str, + compare_version: str, + current_version: str, + restrict_profiles: Optional[list[str]] = None, +) -> tuple[list[RegressionResult], str]: + """Compare current run metrics against a baseline version. + + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. + """ + import pandas as pd + + current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") + baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") + + for col in current_df.columns: + if current_df[col].dtype == object: + current_df[col] = current_df[col].str.strip() + for col in baseline_df.columns: + if baseline_df[col].dtype == object: + baseline_df[col] = baseline_df[col].str.strip() + + if current_df.empty: + return [], "Current run produced no benchmark data" + + model = current_df["model"].iloc[0] + accelerator = current_df["accelerator"].iloc[0] + tp = current_df["TP"].iloc[0] + + logger.info("Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, compare_version, model, accelerator, tp) + + def _assign_profile(row): + try: + isl = int(float(row["prompt toks"])) + osl = int(float(row["output toks"])) + except (ValueError, TypeError): + return None + return PROFILE_MAP.get((isl, osl)) + + current_df["_profile"] = current_df.apply(_assign_profile, axis=1) + baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) + + baseline_filtered = baseline_df[ + (baseline_df["version"] == compare_version) + & (baseline_df["model"] == model) + & (baseline_df["accelerator"] == accelerator) + & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) + ] + + if baseline_filtered.empty: + return [], f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}" + + current_profiles = set(current_df["_profile"].dropna().unique()) + baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) + common_profiles = sorted(current_profiles & baseline_profiles) + + if restrict_profiles: + common_profiles = [p for p in common_profiles if p in restrict_profiles] + + if not common_profiles: + return [], "No common ISL/OSL profiles between current and baseline" + + results: list[RegressionResult] = [] + + for profile in common_profiles: + cur = current_df[current_df["_profile"] == profile].copy() + base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() + + cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") + base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") + + common_conc = set(cur["intended concurrency"].dropna().unique()) & set(base["intended concurrency"].dropna().unique()) + common_conc.discard(1) + + if not common_conc: + continue + + cur_common = cur[cur["intended concurrency"].isin(common_conc)] + base_common = base[base["intended concurrency"].isin(common_conc)] + + for col, meta in METRICS.items(): + if col not in cur_common.columns or col not in base_common.columns: + continue + + cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() + base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() + + cur_gm = geometric_mean(cur_vals) + base_gm = geometric_mean(base_vals) + + if cur_gm is None or base_gm is None: + continue + + pct_diff = ((cur_gm - base_gm) / base_gm) * 100 + threshold = meta["threshold"] + + if meta["higher_is_better"]: + is_regression = pct_diff < -threshold + is_improvement = pct_diff > threshold + else: + is_regression = pct_diff > threshold + is_improvement = pct_diff < -threshold + + results.append(RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + )) + + regressions = [r for r in results if r.is_regression] + logger.info("Comparison complete: %d metric comparisons, %d regressions", + len(results), len(regressions)) + return results, "" + + +def run_regression_analysis( + current_csv_path: Path, + consolidated_csv_path: Path, + compare_version: str, + current_version: str, + output_file: Path, +) -> dict[str, Any]: + """Run regression analysis comparing current run against baseline. + + Returns dict with status, regressions, improvements, and skip_reason. + """ + try: + results, skip_reason = compare_runs( + str(current_csv_path), + str(consolidated_csv_path), + compare_version, + current_version, + ) + + if skip_reason: + analysis = { + "status": "skipped", + "reason": skip_reason, + "current_version": current_version, + "compare_version": compare_version, + } + else: + regressions = [r for r in results if r.is_regression] + improvements = [r for r in results if r.is_improvement] + + analysis = { + "status": "completed", + "current_version": current_version, + "compare_version": compare_version, + "total_comparisons": len(results), + "regression_count": len(regressions), + "improvement_count": len(improvements), + "regressions": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in regressions + ], + "improvements": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in improvements + ], + "all_results": [ + { + "profile": r.profile, + "metric": r.metric_label, + "metric_column": r.metric_column, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + "is_regression": r.is_regression, + "is_improvement": r.is_improvement, + } + for r in results + ], + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis, f, indent=2) + + return analysis + + except Exception as e: + logger.exception("Regression analysis failed") + return {"status": "failed", "error": str(e)} + def run_kpi_analysis( current_kpi_file: Path, diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index ab6f64acc..10415559f 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -16,6 +16,7 @@ GITHUB_APP_PEM_FILE = "topsail-bot.2024-09-18.private-key.pem" GITHUB_APP_CLIENT_ID_FILE = "topsail-bot.clientid" SLACK_TOKEN_FILE = "topsail-bot.slack-token" +SLACK_WEBHOOK_URL_FILE = "slack-webhook-url" DEFAULT_REPO_OWNER = "openshift-psap" DEFAULT_REPO_NAME = "forge" @@ -714,6 +715,156 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok +def _send_via_webhook(webhook_url: str, blocks: list[dict]) -> bool: + """Post a Slack message via Incoming Webhook URL.""" + from urllib.error import URLError + from urllib.request import Request, urlopen + + body = json.dumps({"blocks": blocks}) + try: + req = Request( + webhook_url, + data=body.encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + urlopen(req, timeout=15) # noqa: S310 + return True + except (URLError, OSError) as e: + logger.warning("Slack webhook POST failed: %s", e) + return False + + +def _get_webhook_url(vault_name: str) -> str | None: + """Read the webhook URL from a vault file.""" + from projects.core.library import vault as vault_lib + + path = vault_lib.get_vault_content_path(vault_name, SLACK_WEBHOOK_URL_FILE) + if path is None or not path.exists(): + logger.warning("Webhook URL file not found in vault %s", vault_name) + return None + return path.read_text().strip() + + +def send_regression_notification( + analysis_result: dict, + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, +) -> bool: + """Send regression/improvement Slack notification from analysis results. + + Uses a webhook URL (from *webhook_vault*) when available, falling + back to the Bot-Token path (from *notification_vault*). + + Args: + analysis_result: Output from run_regression_analysis() + model: Model name for display + accelerator: Accelerator name for display + job_id: Job identifier + slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + + Returns: + True if notification sent successfully + """ + status = analysis_result.get("status") + if status == "skipped": + reason = analysis_result.get("reason", "unknown") + logger.info("Regression analysis skipped: %s", reason) + return True + + if status != "completed": + return True + + regressions = analysis_result.get("regressions", []) + improvements = analysis_result.get("improvements", []) + + if not regressions and not improvements: + logger.info("No regressions or improvements to report") + return True + + current_version = analysis_result.get("current_version", "") + compare_version = analysis_result.get("compare_version", "") + + if regressions and improvements: + icon = ":warning:" + headline = "Performance regressions and improvements detected" + elif regressions: + icon = ":warning:" + headline = "Performance regressions detected" + else: + icon = ":large_green_circle:" + headline = "Performance improvements detected" + + detail_lines = [] + all_results = analysis_result.get("all_results", []) + profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + + for profile in profiles: + profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + detail_lines.append(f"\n*{profile}:*") + for r in profile_results: + if r.get("is_regression"): + direction = "dropped" if r["pct_diff"] < 0 else "increased" + detail_lines.append( + f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + else: + detail_lines.append( + f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + + details = "\n".join(detail_lines) + + import re as _re + + if slack_user and _re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + message = ( + f"{icon} *{headline}*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"*Changes:*\n{details}" + ) + + if dry_run: + logger.info("DRY RUN regression notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) + + def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 4965747f1..c2d0676d8 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,3 +36,10 @@ accelerator_env_vars: nvidia: {} amd: VLLM_ROCM_USE_AITER: "1" + +profiler: + enabled: false + s3_bucket: psap-dashboard-data + s3_prefix: pytorch-profiles/rhaiis + vault: psap-forge-dashboard-s3 + labels: [] diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 95231ac5a..113e55d49 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,8 +5,8 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [1, 50, 100, 200, 300] - max_seconds: 450 + rates: [1] + max_seconds: 45 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 6c932e4bb..7323c7d0e 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -1,6 +1,9 @@ vaults: - psap-forge-notifications - psap-forge-mlflow-export +- psap-forge-aws-s3-export +- psap-forge-dashboard-s3 +- psap-forge-rhaiis-slack benchmarks: guidellm: @@ -19,6 +22,11 @@ tests: rhaiis: model_key: qwen3-0_6b workload_key: profile1 + run_benchmark: true + warmup: true + version: "" + compare_version: "" + slack_user: "" caliper: postprocess: @@ -34,6 +42,15 @@ caliper: artifacts_to_kpis: enabled: true output: kpis.json + csv_dashboard: + enabled: false + s3_bucket: psap-dashboard-data + s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv + vault: psap-forge-dashboard-s3 + analyze: + enabled: false + historical_kpis: historical_kpis + output: regression_analysis.json export: from: null diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index d17c4c7d0..44ff22f21 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -67,6 +67,10 @@ def get_test_workload_key() -> str: return config.project.get_config("tests.rhaiis.workload_key") +def get_profiler_config() -> dict: + return dict(config.project.get_config("rhaiis.profiler", {}) or {}) + + def merge_vllm_args( defaults: dict, model: dict, diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 762dbc2b1..b324f7f6f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -112,27 +112,60 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - logger.info("Running benchmark at rates=%s", rates) - - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) - - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}", - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - ) + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + + if profiler_enabled: + logger.info("Skipping warmup -- profiler step provides its own warmup") + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=workload_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + else: + logger.info("Running benchmark at rates=%s", rates) + + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) + + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-{deployment_name}", + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) finally: _capture_and_cleanup(deployment_name, namespace) @@ -141,6 +174,18 @@ def _run_test( except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) + try: + _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args) + except Exception: + logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + + profiler_cfg = runtime_config.get_profiler_config() + if profiler_cfg.get("enabled", False): + try: + _upload_profiler_traces(model_cfg, accelerator, vllm_args, profiler_cfg) + except Exception: + logger.warning("Profiler trace upload failed; continuing", exc_info=True) + try: _set_mlflow_metadata( model_key, @@ -254,6 +299,274 @@ def _generate_psap_payload( ) +def _generate_and_sync_dashboard_csv( + model_cfg: dict, + accelerator: str, + workload_key: str, + vllm_args: dict, +) -> None: + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv + + psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) + if not psap_files: + logger.warning("No PSAP JSON files found, skipping dashboard CSV") + return + + version = config.project.get_config("tests.rhaiis.version", "") + if not version: + logger.info("No version configured, skipping dashboard CSV generation") + return + + csv_path = None + for psap_file in psap_files: + output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" + csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path) + logger.info("Generated dashboard CSV: %s", csv_path) + + if not csv_path: + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): + logger.info("Dashboard CSV S3 sync not enabled") + return + + from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 + + sync_result = sync_csv_to_s3( + csv_path, + s3_bucket=csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data"), + s3_key=csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv"), + vault_name=csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Dashboard CSV sync result: %s", sync_result) + + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not compare_version: + return + + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator) + + +def _run_regression_check( + csv_path, + compare_version: str, + current_version: str, + model_cfg: dict, + accelerator: str, +) -> None: + import tempfile + from pathlib import Path + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.caliper.engine.kpi.analyze import run_regression_analysis + from projects.core.library import config + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") + s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if not credentials_path: + logger.warning("AWS credentials not available, skipping regression analysis") + return + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + except Exception as e: + logger.warning("Could not download consolidated CSV for regression check: %s", e) + return + + try: + output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" + analysis = run_regression_analysis( + current_csv_path=Path(str(csv_path)), + consolidated_csv_path=Path(consolidated_path), + compare_version=compare_version, + current_version=current_version, + output_file=output_file, + ) + + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + from projects.core.notifications.send import send_regression_notification + + send_regression_notification( + analysis, + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=env.ARTIFACT_DIR.rsplit("/", 1)[-1] if env.ARTIFACT_DIR else "", + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-rhaiis-slack", + notification_vault="psap-forge-notifications", + ) + except Exception: + logger.warning("Regression analysis failed; continuing", exc_info=True) + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def _run_warmup_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, + model_cfg: dict, + workload: dict, + benchmark_timeout: int, +) -> None: + """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + + warmup_rates = [200] + warmup_max_seconds = 60 + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=warmup_rates, + max_seconds=warmup_max_seconds, + ) + + logger.info("Running warmup (concurrency=200, duration=60s)") + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-warmup-{deployment_name}", + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + logger.info("Warmup completed") + except Exception: + logger.warning("Warmup failed; continuing with benchmark", exc_info=True) + + +def _run_profiler_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, + model_cfg: dict, + workload: dict, + workload_key: str, + benchmark_timeout: int, +) -> None: + """Run profiler-gated benchmarks: verify prereqs → enable gate → benchmark → disable gate → copy traces.""" + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + from projects.rhaiis.toolbox.copy_profiler_traces.main import run as copy_profiler_traces + from projects.rhaiis.toolbox.enable_profiler_gate.main import run as enable_profiler_gate + from projects.rhaiis.toolbox.verify_profiler_prereqs.main import run as verify_profiler_prereqs + + logger.info("Verifying profiler prerequisites") + verify_profiler_prereqs(namespace=namespace) + + profiler_cfg = runtime_config.get_profiler_config() + labels = profiler_cfg.get("labels", []) + if not labels: + logger.info("No profiler labels configured, skipping profiler step") + return + + profiler_max_seconds = profiler_cfg.get("max_seconds", 60) + + for label in labels: + logger.info("Profiling label=%s", label) + + gate_value = label if isinstance(label, str) else str(label) + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + gate_value=gate_value, + ) + + profiler_rates = profiler_cfg.get("rates", [1]) + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=profiler_rates, + max_seconds=profiler_max_seconds, + ) + + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-profiler-{deployment_name}-{gate_value}", + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + finally: + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + disable=True, + ) + + logger.info("Copying profiler traces from pod") + try: + copy_profiler_traces(name=deployment_name, namespace=namespace) + except Exception: + logger.warning("Failed to copy profiler traces", exc_info=True) + + +def _upload_profiler_traces( + model_cfg: dict, + accelerator: str, + vllm_args: dict, + profiler_cfg: dict, +) -> None: + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 + + traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces" + if not traces_dir.exists() or not any(traces_dir.iterdir()): + logger.info("No profiler traces to upload") + return + + version = config.project.get_config("tests.rhaiis.version", "") + if not version: + logger.info("No version configured, skipping profiler trace upload") + return + + result = upload_profiler_traces_to_s3( + traces_dir, + model_name=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + tp_size=int(vllm_args.get("tensor-parallel-size", 1)), + version=version, + profile_labels=profiler_cfg.get("labels", []), + s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), + s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), + vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Profiler trace upload result: %s", result) + + def _capture_and_cleanup(deployment_name: str, namespace: str) -> None: from projects.rhaiis.toolbox.capture_isvc_state.main import run as capture_isvc_state diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py new file mode 100644 index 000000000..42c56fb7a --- /dev/null +++ b/projects/rhaiis/postprocess/csv_export.py @@ -0,0 +1,184 @@ +"""Generate consolidated_dashboard.csv from PSAP JSON files. + +Matches the CSV schema from model-furnace's visualization/parse_furnace_results.py. +""" + +from __future__ import annotations + +import csv +import json +import logging +import re +from pathlib import Path + +logger = logging.getLogger(__name__) + +FIELDNAMES = [ + "run", "accelerator", "model", "version", "prompt toks", "output toks", "TP", + "measured concurrency", "intended concurrency", "measured rps", + "output_tok/sec", "total_tok/sec", "prompt_token_count_mean", + "prompt_token_count_p99", "output_token_count_mean", + "output_token_count_p99", "ttft_median", "ttft_p95", "ttft_p1", "ttft_p999", + "tpot_median", "tpot_p95", "tpot_p99", "tpot_p999", "tpot_p1", + "itl_median", "itl_p95", "itl_p999", "itl_p1", + "request_latency_median", "request_latency_min", "request_latency_max", + "successful_requests", "errored_requests", "uuid", + "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", + "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", +] + + +def find_psap_files(artifact_dir: Path) -> list[Path]: + """Glob for PSAP_perf*.json files in artifact_dir (recursive).""" + return sorted(artifact_dir.rglob("PSAP_perf*.json")) + + +def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path) -> Path: + """Read a PSAP JSON file and produce a dashboard CSV. + + Args: + psap_json_path: Path to the PSAP payload JSON file. + version: Version string for the CSV rows. + output_path: Where to write the CSV. + + Returns: + Path to the generated CSV file. + """ + with open(psap_json_path, encoding="utf-8") as f: + payload = json.load(f) + + model = payload.get("model", "") + accelerator = payload.get("accelerator_type", "") + tp = payload.get("accelerator_count", 1) + image_tag = payload.get("container_image_tag", "") + runtime_args = json.dumps(payload.get("inference_server_args", {})) + guidellm_start_ms = payload.get("guidellm_start_time_ms") + guidellm_end_ms = payload.get("guidellm_end_time_ms") + + report = payload.get("report", {}) + guidellm_version = report.get("metadata", {}).get("guidellm_version", "") + benchmarks = report.get("benchmarks", []) + + data_str = "" + args = report.get("args", {}) + if isinstance(args, dict): + data_list = args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + prompt_toks = tokens.get("prompt_tokens", "") + output_toks = tokens.get("output_tokens", "") + + rows = [] + for bench in benchmarks: + metrics = bench.get("metrics", {}) + row = _extract_row( + metrics=metrics, + model=model, + accelerator=accelerator, + version=version, + tp=tp, + prompt_toks=prompt_toks, + output_toks=output_toks, + image_tag=image_tag, + runtime_args=runtime_args, + guidellm_start_ms=guidellm_start_ms, + guidellm_end_ms=guidellm_end_ms, + guidellm_version=guidellm_version, + ) + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + logger.info("Generated dashboard CSV with %d rows: %s", len(rows), output_path) + return output_path + + +def _extract_row( + *, + metrics: dict, + model: str, + accelerator: str, + version: str, + tp: int, + prompt_toks: str, + output_toks: str, + image_tag: str, + runtime_args: str, + guidellm_start_ms, + guidellm_end_ms, + guidellm_version: str, +) -> dict: + def _pct(metric_name: str, pct: str) -> float: + return float( + metrics.get(metric_name, {}) + .get("successful", {}) + .get("percentiles", {}) + .get(pct, 0) + ) + + def _stat(metric_name: str, stat: str) -> float: + return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, 0)) + + request_totals = metrics.get("request_totals", {}) + successful = int(request_totals.get("successful", 0)) + errored = int(request_totals.get("errored", 0)) + + concurrency_mean = _stat("request_concurrency", "mean") + intended_concurrency = _stat("request_concurrency", "max") + + output_tok_sec = _stat("output_token_throughput", "mean") + total_tok_sec = _stat("total_token_throughput", "mean") + measured_rps = _stat("request_rate", "mean") + + return { + "run": "", + "accelerator": accelerator, + "model": model, + "version": version, + "prompt toks": prompt_toks, + "output toks": output_toks, + "TP": tp, + "measured concurrency": concurrency_mean, + "intended concurrency": intended_concurrency, + "measured rps": measured_rps, + "output_tok/sec": output_tok_sec, + "total_tok/sec": total_tok_sec, + "prompt_token_count_mean": _stat("prompt_token_count", "mean"), + "prompt_token_count_p99": _pct("prompt_token_count", "p99"), + "output_token_count_mean": _stat("output_token_count", "mean"), + "output_token_count_p99": _pct("output_token_count", "p99"), + "ttft_median": _pct("time_to_first_token_ms", "p50") / 1000.0, + "ttft_p95": _pct("time_to_first_token_ms", "p95") / 1000.0, + "ttft_p1": _pct("time_to_first_token_ms", "p1") / 1000.0, + "ttft_p999": _pct("time_to_first_token_ms", "p999") / 1000.0, + "tpot_median": _pct("time_per_output_token_ms", "p50") / 1000.0, + "tpot_p95": _pct("time_per_output_token_ms", "p95") / 1000.0, + "tpot_p99": _pct("time_per_output_token_ms", "p99") / 1000.0, + "tpot_p999": _pct("time_per_output_token_ms", "p999") / 1000.0, + "tpot_p1": _pct("time_per_output_token_ms", "p1") / 1000.0, + "itl_median": _pct("inter_token_latency_ms", "p50") / 1000.0, + "itl_p95": _pct("inter_token_latency_ms", "p95") / 1000.0, + "itl_p999": _pct("inter_token_latency_ms", "p999") / 1000.0, + "itl_p1": _pct("inter_token_latency_ms", "p1") / 1000.0, + "request_latency_median": _pct("request_latency_ms", "p50") / 1000.0, + "request_latency_min": _stat("request_latency_ms", "min") / 1000.0, + "request_latency_max": _stat("request_latency_ms", "max") / 1000.0, + "successful_requests": successful, + "errored_requests": errored, + "uuid": "", + "ttft_mean": _stat("time_to_first_token_ms", "mean") / 1000.0, + "ttft_p99": _pct("time_to_first_token_ms", "p99") / 1000.0, + "itl_mean": _stat("inter_token_latency_ms", "mean") / 1000.0, + "itl_p99": _pct("inter_token_latency_ms", "p99") / 1000.0, + "runtime_args": runtime_args, + "guidellm_start_time_ms": guidellm_start_ms or "", + "guidellm_end_time_ms": guidellm_end_ms or "", + "image_tag": image_tag, + "guidellm_version": guidellm_version, + } diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py new file mode 100644 index 000000000..efa2c5c3e --- /dev/null +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -0,0 +1,175 @@ +"""S3 dashboard CSV sync and profiler trace upload for rhaiis.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def sync_csv_to_s3( + csv_path: Path, + *, + s3_bucket: str = "psap-dashboard-data", + s3_key: str = "staging/rhaiis-dashboard/consolidated_dashboard.csv", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Download consolidated CSV from S3, append new rows, re-upload. + + Ported from model-furnace's ModelValidator._sync_csv_to_s3(). + """ + try: + import pandas as pd + except ImportError: + return {"status": "failed", "error": "pandas not available"} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + if dry_run: + logger.info("DRY RUN: Would sync %s to s3://%s/%s", csv_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + + try: + s3.download_file(s3_bucket, s3_key, consolidated_path) + logger.info("Downloaded s3://%s/%s", s3_bucket, s3_key) + except Exception as e: + if hasattr(e, "response") and e.response.get("Error", {}).get("Code") == "404": + logger.warning("Consolidated CSV not found on S3, uploading new CSV") + s3.upload_file(str(csv_path), s3_bucket, s3_key) + return {"status": "success", "action": "created", "rows_added": -1} + raise + + new_df = pd.read_csv(csv_path) + if new_df.empty: + logger.warning("No data rows in generated CSV %s", csv_path) + return {"status": "skipped", "reason": "empty CSV"} + + existing_df = pd.read_csv(consolidated_path, on_bad_lines="warn") + merged_df = pd.concat([existing_df, new_df], ignore_index=True) + merged_df.to_csv(consolidated_path, index=False) + rows_added = len(new_df) + logger.info("Appended %d rows (%d total)", rows_added, len(merged_df)) + + s3.upload_file(consolidated_path, s3_bucket, s3_key) + logger.info("Uploaded consolidated CSV to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "action": "appended", "rows_added": rows_added} + + except Exception as e: + logger.error("S3 CSV sync failed: %s", e) + return {"status": "failed", "error": str(e)} + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def upload_profiler_traces_to_s3( + traces_dir: Path, + *, + model_name: str, + accelerator: str, + tp_size: int, + version: str, + profile_labels: list[str], + s3_bucket: str = "psap-dashboard-data", + s3_prefix: str = "pytorch-profiles/rhaiis", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload profiler traces to S3. + + Ported from model-furnace's ModelValidator._upload_profiler_traces_to_s3(). + Path: s3://{bucket}/{prefix}/{accel}/{model}/tp{N}/{version}/{profile_label}/ + """ + if not traces_dir.exists(): + return {"status": "skipped", "reason": f"traces dir not found: {traces_dir}"} + + trace_files = [ + f for f in traces_dir.iterdir() + if f.name.startswith("trace") and "rank0" in f.name + and f.suffix in (".json", ".gz") + ] + if not trace_files: + return {"status": "skipped", "reason": "no trace files found"} + + if dry_run: + logger.info("DRY RUN: Would upload %d traces to S3", len(trace_files)) + return {"status": "success", "dry_run": True, "trace_count": len(trace_files)} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + accel_folder = _accelerator_s3_folder(accelerator) + model_encoded = model_name.replace("/", "--") + uploaded = 0 + + try: + s3 = create_s3_client(credentials_path) + + for trace_file in trace_files: + matched_label = None + for label in profile_labels: + if label in trace_file.name: + matched_label = label + break + if not matched_label: + matched_label = profile_labels[0] if len(profile_labels) == 1 else None + if not matched_label: + logger.warning("Could not match trace %s to a profile label", trace_file.name) + continue + + s3_key = ( + f"{s3_prefix}/{accel_folder}/{model_encoded}" + f"/tp{tp_size}/{version}/{matched_label}/{trace_file.name}" + ) + + try: + s3.head_object(Bucket=s3_bucket, Key=s3_key) + s3.copy_object( + Bucket=s3_bucket, + CopySource={"Bucket": s3_bucket, "Key": s3_key}, + Key=s3_key + ".bak", + ) + logger.info("Backed up existing trace at s3://%s/%s", s3_bucket, s3_key) + except Exception: + pass + + s3.upload_file(str(trace_file), s3_bucket, s3_key) + logger.info("Uploaded trace to s3://%s/%s", s3_bucket, s3_key) + uploaded += 1 + + return {"status": "success", "uploaded": uploaded} + except Exception as e: + logger.error("Trace upload failed: %s", e) + return {"status": "failed", "error": str(e), "uploaded": uploaded} + + +def _accelerator_s3_folder(accelerator: str) -> str: + upper = accelerator.upper() + if upper.startswith("H200"): + return "H200" + if upper.startswith("B200"): + return "B200" + if upper.startswith("MI300"): + return "MI300x" + return accelerator diff --git a/projects/rhaiis/toolbox/copy_profiler_traces/main.py b/projects/rhaiis/toolbox/copy_profiler_traces/main.py new file mode 100644 index 000000000..0d9331aad --- /dev/null +++ b/projects/rhaiis/toolbox/copy_profiler_traces/main.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str): + return execute_tasks(locals()) + + +@task +def setup_directories(args, context): + shell.mkdir("artifacts/traces") + return "Traces directory created" + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def list_trace_files(args, context): + result = shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'ls /tmp/trace_*.json* 2>/dev/null || echo NO_TRACES'", + check=False, + log_stdout=False, + ) + + if "NO_TRACES" in result.stdout or not result.stdout.strip(): + raise RuntimeError(f"No profiler traces found in pod {context.pod_name}") + + trace_list = result.stdout.strip() + context.trace_count = len(trace_list.splitlines()) + return f"Found {context.trace_count} trace files" + + +@task +def copy_traces(args, context): + traces_dir = args.artifact_dir / "artifacts/traces" + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'cd /tmp && tar cf - trace_*.json*' " + f"| tar --no-same-owner -xf - -C {traces_dir}", + ) + return f"Copied {context.trace_count} trace files to {traces_dir}" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py new file mode 100644 index 000000000..69468b405 --- /dev/null +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str, gate_value: str = "0", disable: bool = False): + return execute_tasks(locals()) + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def toggle_profiler_gate(args, context): + if args.disable: + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'rm -f /tmp/profiler_gate'", + check=False, + ) + return f"Profiler gate disabled on {context.pod_name}" + + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + f"-- sh -c 'echo {args.gate_value} > /tmp/profiler_gate'", + ) + return f"Profiler gate enabled on {context.pod_name} (value={args.gate_value})" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py new file mode 100644 index 000000000..dc2e95e50 --- /dev/null +++ b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Verify that the vLLM profiler webhook infrastructure is ready. + +Ported from model-furnace's oc_commands.verify_profiler_prerequisites(). +Checks webhook pod, MutatingWebhookConfiguration, TLS secret, and +ConfigMap before any profiler run is attempted. +""" + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + +MIN_CA_BUNDLE_LENGTH = 100 + + +@entrypoint +def run(*, namespace: str): + return execute_tasks(locals()) + + +@task +def check_webhook_pod(args, context): + result = shell.run( + "oc get pods -n vllm-profiler -l app=env-injector " + "-o jsonpath='{.items[0].status.phase}'", + check=False, + ) + phase = result.stdout.strip().strip("'") + if result.returncode != 0 or phase != "Running": + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook pod not running (status: {phase or 'not found'}). " + "Deploy it with: ./scripts/deploy.sh" + ) + return f"FAIL: webhook pod status={phase or 'not found'}" + return "OK: webhook pod is Running" + + +@task +def check_webhook_target_namespace(args, context): + result = shell.run( + "oc get deployment env-injector -n vllm-profiler " + "-o jsonpath='{.spec.template.spec.containers[0]" + '.env[?(@.name=="TARGET_NAMESPACE")].value}\'', + check=False, + ) + target_ns = result.stdout.strip().strip("'") + if result.returncode != 0 or not target_ns: + context.errors = getattr(context, "errors", []) + context.errors.append("Could not read webhook TARGET_NAMESPACE from deployment") + return "FAIL: could not read TARGET_NAMESPACE" + if target_ns != args.namespace: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook TARGET_NAMESPACE is '{target_ns}' but deployment namespace " + f"is '{args.namespace}'. Update manifests.yaml and run: " + "oc apply -f manifests.yaml" + ) + return f"FAIL: TARGET_NAMESPACE={target_ns} != {args.namespace}" + return f"OK: TARGET_NAMESPACE matches '{args.namespace}'" + + +@task +def check_mutating_webhook(args, context): + result = shell.run( + "oc get mutatingwebhookconfiguration env-injector-webhook " + "-o jsonpath='{.webhooks[0].clientConfig.caBundle}'", + check=False, + ) + ca_bundle = result.stdout.strip().strip("'") + if result.returncode != 0 or len(ca_bundle) < MIN_CA_BUNDLE_LENGTH: + context.errors = getattr(context, "errors", []) + context.errors.append( + "MutatingWebhookConfiguration missing or caBundle is empty. " + "Run: bash scripts/gen-certs.sh && bash scripts/patch-ca-bundle.sh" + ) + return "FAIL: MutatingWebhookConfiguration missing or bad caBundle" + return "OK: MutatingWebhookConfiguration has valid caBundle" + + +@task +def check_tls_secret(args, context): + result = shell.run( + "oc get secret env-injector-certs -n vllm-profiler -o name", + check=False, + ) + if result.returncode != 0: + context.errors = getattr(context, "errors", []) + context.errors.append( + "TLS secret 'env-injector-certs' not found in vllm-profiler namespace" + ) + return "FAIL: TLS secret missing" + return "OK: TLS secret exists" + + +@task +def check_configmap(args, context): + result = shell.run( + f"oc get configmap env-injector-files -n {args.namespace} " + "-o jsonpath='{.data}'", + check=False, + ) + if result.returncode != 0 or "sitecustomize.py" not in result.stdout: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"ConfigMap 'env-injector-files' not found in {args.namespace}. " + "Create it with: oc apply -k " + ) + return f"FAIL: ConfigMap missing in {args.namespace}" + return f"OK: ConfigMap exists in {args.namespace}" + + +@task +def raise_on_errors(args, context): + errors = getattr(context, "errors", []) + if errors: + raise ValueError( + "Profiler prerequisites not met:\n - " + "\n - ".join(errors) + ) + return "All profiler prerequisites verified" + + +if __name__ == "__main__": + run.main() diff --git a/vaults/psap-forge-dashboard-s3.yaml b/vaults/psap-forge-dashboard-s3.yaml new file mode 100644 index 000000000..145fc0e70 --- /dev/null +++ b/vaults/psap-forge-dashboard-s3.yaml @@ -0,0 +1,16 @@ +env_key: PSAP_FORGE_DASHBOARD_S3_SECRET_PATH +description: | + AWS credentials for rhaiis dashboard CSV sync (psap-dashboard-data) + and profiler trace uploads. Ported from model-furnace s3-uploader.conf. + +content: + aws.credentials: + description: | + AWS credentials file (INI format). The key must have + s3:GetObject, s3:PutObject, s3:CopyObject on: + - psap-dashboard-data (CSV dashboard, profiler traces) + - psap-model-furnace (vLLM logs) + sample: | + [default] + aws_access_key_id = AKIA... + aws_secret_access_key = ..... diff --git a/vaults/psap-forge-rhaiis-slack.yaml b/vaults/psap-forge-rhaiis-slack.yaml new file mode 100644 index 000000000..838d41964 --- /dev/null +++ b/vaults/psap-forge-rhaiis-slack.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_FORGE_RHAIIS_SLACK_SECRET_PATH +description: | + Slack Incoming Webhook for rhaiis regression/comparison alerts. + Ported from model-furnace's slack.conf. + +content: + slack-webhook-url: + description: | + Plain-text file containing a single Slack Incoming Webhook URL. + Example: https://hooks.slack.com/services/T.../B.../... From 579b8aa1a0f3fd7ff9828a2e900e7dcceb384ea8 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 19:44:26 -0400 Subject: [PATCH 0084/1072] Fix CSV schema alignment with model-furnace and add run UUID - Fix CSV export to match model-furnace field extraction exactly: accelerator parsing, runtime_args formatting, metric key paths, raw ms values (no /1000), median/p01 instead of p50/p1, and add missing columns (DP, dataset, spec_decoding, etc.) - Generate a single UUID4 per run and thread it through PSAP payload (experiment_id), dashboard CSV (uuid column), predictor log upload (logs/{uuid}.log on S3), and regression notifications (job_id) - Add predictor pod log upload to s3://psap-model-furnace/logs/ Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 50 +++++- projects/rhaiis/postprocess/csv_export.py | 180 +++++++++++++------- projects/rhaiis/postprocess/parser.py | 3 +- projects/rhaiis/postprocess/s3_dashboard.py | 35 ++++ 4 files changed, 199 insertions(+), 69 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b324f7f6f..52b65f02f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import uuid as _uuid_mod from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels @@ -48,6 +49,9 @@ def _run_test( namespace: str, deployment_name: str | None = None, ) -> int: + run_uuid = str(_uuid_mod.uuid4()) + logger.info("Run UUID: %s", run_uuid) + model_cfg = runtime_config.get_model(model_key) workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() @@ -170,15 +174,20 @@ def _run_test( _capture_and_cleanup(deployment_name, namespace) try: - _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key) + _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args) + _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + try: + _upload_predictor_log(run_uuid) + except Exception: + logger.warning("Predictor log upload failed; continuing", exc_info=True) + profiler_cfg = runtime_config.get_profiler_config() if profiler_cfg.get("enabled", False): try: @@ -266,6 +275,8 @@ def _generate_psap_payload( vllm_image: str, vllm_args: dict, workload_key: str, + *, + run_uuid: str = "", ) -> None: from pathlib import Path @@ -288,6 +299,7 @@ def _generate_psap_payload( vllm_args=vllm_args, accelerator=accelerator, workload_key=workload_key, + run_uuid=run_uuid, ) output_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "results" write_psap_payload( @@ -304,6 +316,8 @@ def _generate_and_sync_dashboard_csv( accelerator: str, workload_key: str, vllm_args: dict, + *, + run_uuid: str = "", ) -> None: from pathlib import Path @@ -323,7 +337,7 @@ def _generate_and_sync_dashboard_csv( csv_path = None for psap_file in psap_files: output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" - csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path) + csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path, run_uuid=run_uuid) logger.info("Generated dashboard CSV: %s", csv_path) if not csv_path: @@ -349,7 +363,7 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator) + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) def _run_regression_check( @@ -358,6 +372,8 @@ def _run_regression_check( current_version: str, model_cfg: dict, accelerator: str, + *, + run_uuid: str = "", ) -> None: import tempfile from pathlib import Path @@ -403,7 +419,7 @@ def _run_regression_check( analysis, model=model_cfg.get("hf_model_id", ""), accelerator=accelerator, - job_id=env.ARTIFACT_DIR.rsplit("/", 1)[-1] if env.ARTIFACT_DIR else "", + job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", @@ -416,6 +432,30 @@ def _run_regression_check( os.unlink(consolidated_path) +def _upload_predictor_log(run_uuid: str) -> None: + """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 + + log_path = Path(env.ARTIFACT_DIR) / "artifacts" / "inferenceservice.pods.logs" + if not log_path.exists(): + logger.info("No predictor pod log found at %s, skipping upload", log_path) + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + result = upload_predictor_log_to_s3( + log_path, + run_uuid=run_uuid, + vault_name=vault_name, + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Predictor log upload result: %s", result) + + def _run_warmup_step( *, deployment_name: str, diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py index 42c56fb7a..93270ac27 100644 --- a/projects/rhaiis/postprocess/csv_export.py +++ b/projects/rhaiis/postprocess/csv_export.py @@ -25,35 +25,65 @@ "successful_requests", "errored_requests", "uuid", "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", + "DP", "dataset", "spec_decoding", "prefix_caching", + "turns", "prefix_tokens", "prefix_count", "request_type", ] +def _parse_accelerator_key(acc_key: str) -> tuple[str, str | None]: + """Return (chip_family, cluster_tag) from an accelerator key. + + Convention: keys follow ``_`` (e.g. H200_HERA). + Keys without an underscore have no cluster tag. + """ + if not acc_key: + return "", None + if "_" in acc_key: + chip_family, cluster_tag = acc_key.split("_", 1) + return chip_family, cluster_tag.lower() + return acc_key, None + + +def _format_runtime_args(inference_server_args: dict, trtllm_config: dict | None = None) -> str: + """Format args into semicolon-separated ``key: value`` string matching model-furnace.""" + if not inference_server_args and not trtllm_config: + return "" + parts = [] + for key, value in (inference_server_args or {}).items(): + formatted_key = key.replace("_", "-") + formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + parts.append(f"{formatted_key}: {formatted_value}") + for key, value in (trtllm_config or {}).items(): + formatted_key = f"trtllm.{key.replace('_', '-')}" + formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + parts.append(f"{formatted_key}: {formatted_value}") + return "; ".join(parts) + + def find_psap_files(artifact_dir: Path) -> list[Path]: """Glob for PSAP_perf*.json files in artifact_dir (recursive).""" return sorted(artifact_dir.rglob("PSAP_perf*.json")) -def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path) -> Path: - """Read a PSAP JSON file and produce a dashboard CSV. - - Args: - psap_json_path: Path to the PSAP payload JSON file. - version: Version string for the CSV rows. - output_path: Where to write the CSV. - - Returns: - Path to the generated CSV file. - """ +def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "") -> Path: + """Read a PSAP JSON file and produce a dashboard CSV.""" with open(psap_json_path, encoding="utf-8") as f: payload = json.load(f) model = payload.get("model", "") - accelerator = payload.get("accelerator_type", "") - tp = payload.get("accelerator_count", 1) + acc_key = payload.get("accelerator_type", "") + acc, cluster_tag = _parse_accelerator_key(acc_key) + + inference_server_args = payload.get("inference_server_args", {}) + tp = ( + inference_server_args.get("tensor-parallel-size") + or inference_server_args.get("tp-size") + or inference_server_args.get("tp_size") + or payload.get("accelerator_count", 1) + ) + image_tag = payload.get("container_image_tag", "") - runtime_args = json.dumps(payload.get("inference_server_args", {})) - guidellm_start_ms = payload.get("guidellm_start_time_ms") - guidellm_end_ms = payload.get("guidellm_end_time_ms") + runtime_args = _format_runtime_args(inference_server_args, payload.get("trtllm_config")) report = payload.get("report", {}) guidellm_version = report.get("metadata", {}).get("guidellm_version", "") @@ -67,16 +97,32 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path data_str = data_list[0] if isinstance(data_list, list) else str(data_list) tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) - prompt_toks = tokens.get("prompt_tokens", "") - output_toks = tokens.get("output_tokens", "") + prompt_toks = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + output_toks = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + start_times = [] + end_times = [] + for bench in benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + guidellm_start_ms = int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") + guidellm_end_ms = int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") rows = [] for bench in benchmarks: metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + model_name = f"{acc}-{cluster_tag}-{model}-{tp}" if cluster_tag else f"{acc}-{model}-{tp}" + row = _extract_row( metrics=metrics, + strategy=strategy, + model_name=model_name, model=model, - accelerator=accelerator, + accelerator=acc, version=version, tp=tp, prompt_toks=prompt_toks, @@ -86,6 +132,7 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path guidellm_start_ms=guidellm_start_ms, guidellm_end_ms=guidellm_end_ms, guidellm_version=guidellm_version, + run_uuid=run_uuid, ) rows.append(row) @@ -102,83 +149,90 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path def _extract_row( *, metrics: dict, + strategy: dict, + model_name: str, model: str, accelerator: str, version: str, - tp: int, - prompt_toks: str, - output_toks: str, + tp, + prompt_toks, + output_toks, image_tag: str, runtime_args: str, guidellm_start_ms, guidellm_end_ms, guidellm_version: str, + run_uuid: str = "", ) -> dict: - def _pct(metric_name: str, pct: str) -> float: - return float( + def _pct(metric_name: str, pct: str): + return ( metrics.get(metric_name, {}) .get("successful", {}) .get("percentiles", {}) - .get(pct, 0) + .get(pct) ) - def _stat(metric_name: str, stat: str) -> float: - return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, 0)) + def _stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("successful", {}).get(stat) - request_totals = metrics.get("request_totals", {}) - successful = int(request_totals.get("successful", 0)) - errored = int(request_totals.get("errored", 0)) - - concurrency_mean = _stat("request_concurrency", "mean") - intended_concurrency = _stat("request_concurrency", "max") + def _total_stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("total", {}).get(stat) - output_tok_sec = _stat("output_token_throughput", "mean") - total_tok_sec = _stat("total_token_throughput", "mean") - measured_rps = _stat("request_rate", "mean") + request_totals = metrics.get("request_totals", {}) + successful = request_totals.get("successful", 0) + errored = request_totals.get("errored", 0) return { - "run": "", + "run": model_name, "accelerator": accelerator, "model": model, "version": version, "prompt toks": prompt_toks, "output toks": output_toks, "TP": tp, - "measured concurrency": concurrency_mean, - "intended concurrency": intended_concurrency, - "measured rps": measured_rps, - "output_tok/sec": output_tok_sec, - "total_tok/sec": total_tok_sec, + "measured concurrency": _stat("request_concurrency", "mean"), + "intended concurrency": strategy.get("streams"), + "measured rps": _stat("requests_per_second", "mean"), + "output_tok/sec": _total_stat("output_tokens_per_second", "mean"), + "total_tok/sec": _total_stat("tokens_per_second", "mean"), "prompt_token_count_mean": _stat("prompt_token_count", "mean"), "prompt_token_count_p99": _pct("prompt_token_count", "p99"), "output_token_count_mean": _stat("output_token_count", "mean"), "output_token_count_p99": _pct("output_token_count", "p99"), - "ttft_median": _pct("time_to_first_token_ms", "p50") / 1000.0, - "ttft_p95": _pct("time_to_first_token_ms", "p95") / 1000.0, - "ttft_p1": _pct("time_to_first_token_ms", "p1") / 1000.0, - "ttft_p999": _pct("time_to_first_token_ms", "p999") / 1000.0, - "tpot_median": _pct("time_per_output_token_ms", "p50") / 1000.0, - "tpot_p95": _pct("time_per_output_token_ms", "p95") / 1000.0, - "tpot_p99": _pct("time_per_output_token_ms", "p99") / 1000.0, - "tpot_p999": _pct("time_per_output_token_ms", "p999") / 1000.0, - "tpot_p1": _pct("time_per_output_token_ms", "p1") / 1000.0, - "itl_median": _pct("inter_token_latency_ms", "p50") / 1000.0, - "itl_p95": _pct("inter_token_latency_ms", "p95") / 1000.0, - "itl_p999": _pct("inter_token_latency_ms", "p999") / 1000.0, - "itl_p1": _pct("inter_token_latency_ms", "p1") / 1000.0, - "request_latency_median": _pct("request_latency_ms", "p50") / 1000.0, - "request_latency_min": _stat("request_latency_ms", "min") / 1000.0, - "request_latency_max": _stat("request_latency_ms", "max") / 1000.0, + "ttft_median": _stat("time_to_first_token_ms", "median"), + "ttft_p95": _pct("time_to_first_token_ms", "p95"), + "ttft_p1": _pct("time_to_first_token_ms", "p01"), + "ttft_p999": _pct("time_to_first_token_ms", "p999"), + "tpot_median": _stat("time_per_output_token_ms", "median"), + "tpot_p95": _pct("time_per_output_token_ms", "p95"), + "tpot_p99": _pct("time_per_output_token_ms", "p99"), + "tpot_p999": _pct("time_per_output_token_ms", "p999"), + "tpot_p1": _pct("time_per_output_token_ms", "p01"), + "itl_median": _stat("inter_token_latency_ms", "median"), + "itl_p95": _pct("inter_token_latency_ms", "p95"), + "itl_p999": _pct("inter_token_latency_ms", "p999"), + "itl_p1": _pct("inter_token_latency_ms", "p01"), + "request_latency_median": _stat("request_latency", "median"), + "request_latency_min": _stat("request_latency", "min"), + "request_latency_max": _stat("request_latency", "max"), "successful_requests": successful, "errored_requests": errored, - "uuid": "", - "ttft_mean": _stat("time_to_first_token_ms", "mean") / 1000.0, - "ttft_p99": _pct("time_to_first_token_ms", "p99") / 1000.0, - "itl_mean": _stat("inter_token_latency_ms", "mean") / 1000.0, - "itl_p99": _pct("inter_token_latency_ms", "p99") / 1000.0, + "uuid": run_uuid, + "ttft_mean": _stat("time_to_first_token_ms", "mean"), + "ttft_p99": _pct("time_to_first_token_ms", "p99"), + "itl_mean": _stat("inter_token_latency_ms", "mean"), + "itl_p99": _pct("inter_token_latency_ms", "p99"), "runtime_args": runtime_args, "guidellm_start_time_ms": guidellm_start_ms or "", "guidellm_end_time_ms": guidellm_end_ms or "", "image_tag": image_tag, "guidellm_version": guidellm_version, + "DP": "", + "dataset": "", + "spec_decoding": "", + "prefix_caching": "", + "turns": "", + "prefix_tokens": "", + "prefix_count": "", + "request_type": "", } diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 31059e2ea..c0151a298 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -117,6 +117,7 @@ def generate_psap_payload( vllm_args: dict[str, Any], accelerator: str, workload_key: str, + run_uuid: str = "", ) -> dict[str, Any]: report = json.loads(benchmarks_json_path.read_text(encoding="utf-8")) @@ -135,7 +136,7 @@ def generate_psap_payload( guidellm_end = int(max(ends) * 1000) return { - "experiment_id": str(uuid.uuid4()).upper(), + "experiment_id": run_uuid or str(uuid.uuid4()).upper(), "experiment_type": "perf", "model": model_id, "inference_server": "vllm", diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index efa2c5c3e..253c176b8 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -164,6 +164,41 @@ def upload_profiler_traces_to_s3( return {"status": "failed", "error": str(e), "uploaded": uploaded} +def upload_predictor_log_to_s3( + log_path: Path, + *, + run_uuid: str, + s3_bucket: str = "psap-model-furnace", + vault_name: str = "psap-forge-dashboard-s3", + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload the vLLM predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + if not log_path.exists(): + return {"status": "skipped", "reason": f"log not found: {log_path}"} + + s3_key = f"logs/{run_uuid}.log" + + if dry_run: + logger.info("DRY RUN: Would upload %s to s3://%s/%s", log_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + try: + s3 = create_s3_client(credentials_path) + s3.upload_file(str(log_path), s3_bucket, s3_key) + logger.info("Uploaded predictor log to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "s3_key": s3_key} + except Exception as e: + logger.error("Predictor log upload failed: %s", e) + return {"status": "failed", "error": str(e)} + + def _accelerator_s3_folder(accelerator: str) -> str: upper = accelerator.upper() if upper.startswith("H200"): From 373743da1ee0cefe9ad7700ce7d0125a2f1630bf Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 20:16:00 -0400 Subject: [PATCH 0085/1072] Fix: keep kubectl in forge-core container image The openshift-client tarball includes both oc and kubectl, but the Containerfile was deleting kubectl after extraction. Pipeline tasks that reference kubectl (e.g. 00-pre-cleanup) fail with exit code 127. Co-authored-by: Cursor --- projects/core/image/Containerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/core/image/Containerfile b/projects/core/image/Containerfile index 200c61e53..50935add0 100644 --- a/projects/core/image/Containerfile +++ b/projects/core/image/Containerfile @@ -37,9 +37,9 @@ RUN pip install --no-cache-dir uv # Install OpenShift CLI (oc) RUN curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz" && \ tar -xzf openshift-client-linux.tar.gz && \ - chmod +x oc && \ - mv oc /usr/local/bin/ && \ - rm -f openshift-client-linux.tar.gz kubectl + chmod +x oc kubectl && \ + mv oc kubectl /usr/local/bin/ && \ + rm -f openshift-client-linux.tar.gz # Optimize uv for container environments ENV UV_COMPILE_BYTECODE=1 From fd8edec3613d21a91b57c3c80acc455637587015 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 12 Jul 2026 20:19:31 -0400 Subject: [PATCH 0086/1072] Revert "Fix: keep kubectl in forge-core container image" This reverts commit e17f1712338e68a29db7f0f13db20be0692efd8c. --- projects/core/image/Containerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/core/image/Containerfile b/projects/core/image/Containerfile index 50935add0..200c61e53 100644 --- a/projects/core/image/Containerfile +++ b/projects/core/image/Containerfile @@ -37,9 +37,9 @@ RUN pip install --no-cache-dir uv # Install OpenShift CLI (oc) RUN curl -LO "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz" && \ tar -xzf openshift-client-linux.tar.gz && \ - chmod +x oc kubectl && \ - mv oc kubectl /usr/local/bin/ && \ - rm -f openshift-client-linux.tar.gz + chmod +x oc && \ + mv oc /usr/local/bin/ && \ + rm -f openshift-client-linux.tar.gz kubectl # Optimize uv for container environments ENV UV_COMPILE_BYTECODE=1 From 3d3de4e0ba5e3636f7ec42968634a01ed54b5b7a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:28:37 -0400 Subject: [PATCH 0087/1072] Fix accelerator label and exclude warmup from PSAP payload - Use gpu_type (e.g. H200) instead of platform name (nvidia) for PSAP payload, CSV, and S3 artifact paths - Exclude warmup benchmarks.json from PSAP payload generation to prevent warmup results contaminating the dashboard CSV Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 52b65f02f..da7c1688f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -55,6 +55,7 @@ def _run_test( model_cfg = runtime_config.get_model(model_key) workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator deploy_cfg = runtime_config.get_deploy_config() benchmark_cfg = runtime_config.get_benchmark_config() @@ -174,12 +175,12 @@ def _run_test( _capture_and_cleanup(deployment_name, namespace) try: - _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) + _generate_psap_payload(model_cfg, gpu_type, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator, workload_key, vllm_args, run_uuid=run_uuid) + _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) @@ -191,7 +192,7 @@ def _run_test( profiler_cfg = runtime_config.get_profiler_config() if profiler_cfg.get("enabled", False): try: - _upload_profiler_traces(model_cfg, accelerator, vllm_args, profiler_cfg) + _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: logger.warning("Profiler trace upload failed; continuing", exc_info=True) @@ -282,15 +283,16 @@ def _generate_psap_payload( from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - matches = list( + all_matches = list( Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") ) + matches = [m for m in all_matches if "warmup" not in str(m)] if not matches: logger.warning( "benchmarks.json not found under %s, skipping PSAP payload", env.ARTIFACT_DIR ) return - benchmarks_json = matches[0] + benchmarks_json = matches[-1] payload = generate_psap_payload( benchmarks_json_path=benchmarks_json, From 718c8eadc9f34eeee8f599bb0c1d67de4d50f5d2 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:39:40 -0400 Subject: [PATCH 0088/1072] Only generate PSAP payload from main benchmark results Track the main benchmark artifact directory explicitly by index instead of globbing for all benchmarks.json files. Warmup and profiler benchmark results are no longer included in the PSAP payload or dashboard CSV. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 44 ++++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index da7c1688f..67f910c93 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -147,6 +147,7 @@ def _run_test( benchmark_timeout=benchmark_timeout, ) + main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") else: @@ -162,6 +163,7 @@ def _run_test( max_seconds=max_seconds, ) + pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", name=f"guidellm-{deployment_name}", @@ -171,18 +173,26 @@ def _run_test( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, ) + from pathlib import Path + candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) + if candidates: + main_benchmark_dir = candidates[0] finally: _capture_and_cleanup(deployment_name, namespace) - try: - _generate_psap_payload(model_cfg, gpu_type, vllm_image, vllm_args, workload_key, run_uuid=run_uuid) - except Exception: - logger.warning("PSAP payload generation failed; continuing", exc_info=True) + if main_benchmark_dir: + try: + _generate_psap_payload( + model_cfg, gpu_type, vllm_image, vllm_args, workload_key, + run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, + ) + except Exception: + logger.warning("PSAP payload generation failed; continuing", exc_info=True) - try: - _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + try: + _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) try: _upload_predictor_log(run_uuid) @@ -278,21 +288,25 @@ def _generate_psap_payload( workload_key: str, *, run_uuid: str = "", + benchmark_dir: Path | None = None, ) -> None: from pathlib import Path from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - all_matches = list( - Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") - ) - matches = [m for m in all_matches if "warmup" not in str(m)] - if not matches: + if benchmark_dir: + benchmarks_json = benchmark_dir / "artifacts" / "results" / "benchmarks.json" + else: + matches = list( + Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") + ) + benchmarks_json = matches[-1] if matches else None + + if not benchmarks_json or not benchmarks_json.exists(): logger.warning( - "benchmarks.json not found under %s, skipping PSAP payload", env.ARTIFACT_DIR + "benchmarks.json not found, skipping PSAP payload" ) return - benchmarks_json = matches[-1] payload = generate_psap_payload( benchmarks_json_path=benchmarks_json, From f0add8b790858b38505cb7177009d618a5101bc5 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 11:53:20 -0400 Subject: [PATCH 0089/1072] Add cluster_tag config for run ID in dashboard CSV When rhaiis.cluster_tag is set (e.g. "hera2"), the PSAP payload accelerator_type becomes "H200_HERA2", which csv_export splits into accelerator=H200 and run=H200-hera2-model-TP. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index c2d0676d8..2116f1acf 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -1,5 +1,6 @@ namespace: forge-rhaiis accelerator: nvidia +cluster_tag: "" gpu_types: nvidia: h200 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 67f910c93..83efeacc4 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -56,6 +56,9 @@ def _run_test( workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + from projects.core.library import config as _cfg + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() deploy_cfg = runtime_config.get_deploy_config() benchmark_cfg = runtime_config.get_benchmark_config() @@ -183,14 +186,14 @@ def _run_test( if main_benchmark_dir: try: _generate_psap_payload( - model_cfg, gpu_type, vllm_image, vllm_args, workload_key, + model_cfg, accelerator_key, vllm_image, vllm_args, workload_key, run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, ) except Exception: logger.warning("PSAP payload generation failed; continuing", exc_info=True) try: - _generate_and_sync_dashboard_csv(model_cfg, gpu_type, workload_key, vllm_args, run_uuid=run_uuid) + _generate_and_sync_dashboard_csv(model_cfg, accelerator_key, workload_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) From 01e1209de06ff04189f5359a11538d532962a57e Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 13:57:35 -0400 Subject: [PATCH 0090/1072] Fix vllm_args merge order, default configs, and log upload bucket - merge_vllm_args now applies rhaiis.vllm_args last so configOverrides like tensor-parallel-size take precedence over model defaults - Remove tensor-parallel-size from rhaiis.yaml defaults (models define it) - Move namespace and image_pull_secret into rhaiis.yaml defaults - Fix predictor log upload bucket to psap-dashboard-data Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 8 +++----- projects/rhaiis/orchestration/runtime_config.py | 6 +++--- projects/rhaiis/postprocess/s3_dashboard.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 2116f1acf..4a43129e7 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -1,4 +1,4 @@ -namespace: forge-rhaiis +namespace: kserve-e2e-perf accelerator: nvidia cluster_tag: "" @@ -16,8 +16,8 @@ deploy: cpu_request: "4" memory_request: "16Gi" storage_source: hf - storage_pvc: model-pvc-2 - image_pull_secret: "" + storage_pvc: model-pvc + image_pull_secret: npalaska-image-pull service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 @@ -28,8 +28,6 @@ vllm_args: no-enable-log-requests: true no-enable-prefix-caching: true uvicorn-log-level: debug - max-model-len: 8192 - tensor-parallel-size: 1 env_vars: {} diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 44ff22f21..1abbc2492 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -72,13 +72,13 @@ def get_profiler_config() -> dict: def merge_vllm_args( - defaults: dict, + overrides: dict, model: dict, workload: dict, ) -> dict: - merged = dict(defaults) - merged.update(model.get("vllm_args", {})) + merged = dict(model.get("vllm_args", {})) merged.update(workload.get("vllm_args", {})) + merged.update(overrides) return merged diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index 253c176b8..d8b3705a0 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -168,7 +168,7 @@ def upload_predictor_log_to_s3( log_path: Path, *, run_uuid: str, - s3_bucket: str = "psap-model-furnace", + s3_bucket: str = "psap-dashboard-data", vault_name: str = "psap-forge-dashboard-s3", credentials_file: str = "aws.credentials", dry_run: bool = False, From d3a33660761555faf460f5309324a0d9d9348f70 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 14:17:55 -0400 Subject: [PATCH 0091/1072] Fix predictor log path to match toolbox task artifact dir The capture_isvc_state toolbox task writes logs under its own sub-directory (NNN__capture_isvc_state/artifacts/), not the top-level ARTIFACT_DIR. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 83efeacc4..f638572ee 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -458,9 +458,10 @@ def _upload_predictor_log(run_uuid: str) -> None: from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 - log_path = Path(env.ARTIFACT_DIR) / "artifacts" / "inferenceservice.pods.logs" - if not log_path.exists(): - logger.info("No predictor pod log found at %s, skipping upload", log_path) + matches = sorted(Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs")) + log_path = matches[-1] if matches else None + if not log_path or not log_path.exists(): + logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) return csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) From 542d404a59f84fe321128f611ee7df93c80d6d76 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 15:27:25 -0400 Subject: [PATCH 0092/1072] Make warmup and profiler configs explicit and match model-furnace - Add rhaiis.warmup config section (rate: 200, max_seconds: 60) - Add profiler rates/max_seconds to config (200/200, matching model-furnace) - Read warmup params from config instead of hardcoding Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 6 ++++++ projects/rhaiis/orchestration/test_phase.py | 11 +++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 4a43129e7..96f1d7def 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,9 +36,15 @@ accelerator_env_vars: amd: VLLM_ROCM_USE_AITER: "1" +warmup: + rate: 200 + max_seconds: 60 + profiler: enabled: false s3_bucket: psap-dashboard-data s3_prefix: pytorch-profiles/rhaiis vault: psap-forge-dashboard-s3 labels: [] + rates: [200] + max_seconds: 200 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f638572ee..c641cb21b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -491,18 +491,21 @@ def _run_warmup_step( run as run_guidellm_benchmark, ) - warmup_rates = [200] - warmup_max_seconds = 60 + from projects.core.library import config + + warmup_cfg = config.project.get_config("rhaiis.warmup", {}) + warmup_rate = warmup_cfg.get("rate", 200) + warmup_max_seconds = warmup_cfg.get("max_seconds", 60) guidellm_args = runtime_config.build_guidellm_args( benchmark_cfg=benchmark_cfg, model_id=model_cfg["hf_model_id"], data=workload["data"], - rates=warmup_rates, + rates=[warmup_rate], max_seconds=warmup_max_seconds, ) - logger.info("Running warmup (concurrency=200, duration=60s)") + logger.info("Running warmup (concurrency=%d, duration=%ds)", warmup_rate, warmup_max_seconds) try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", From 6cf58cb3b0abe5a77ef5453381bc86127249df97 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 15:49:24 -0400 Subject: [PATCH 0093/1072] Auto-generate profiler labels from workload data Derives labels like isl1000_osl1000 from the workload's data field instead of requiring manual rhaiis.profiler.labels config, matching model-furnace's get_profile_label() behavior. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index c641cb21b..ef119d5d9 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -546,8 +546,8 @@ def _run_profiler_step( profiler_cfg = runtime_config.get_profiler_config() labels = profiler_cfg.get("labels", []) if not labels: - logger.info("No profiler labels configured, skipping profiler step") - return + labels = [_derive_profiler_label(workload)] + logger.info("Auto-generated profiler label from workload: %s", labels[0]) profiler_max_seconds = profiler_cfg.get("max_seconds", 60) @@ -594,6 +594,15 @@ def _run_profiler_step( logger.warning("Failed to copy profiler traces", exc_info=True) +def _derive_profiler_label(workload: dict) -> str: + """Auto-generate a profiler label like 'isl1000_osl1000' from the workload data string.""" + data = workload.get("data", "") + params = dict(item.split("=", 1) for item in data.split(",") if "=" in item) + isl = params.get("prompt_tokens", "0") + osl = params.get("output_tokens", "0") + return f"isl{isl}_osl{osl}" + + def _upload_profiler_traces( model_cfg: dict, accelerator: str, From 8cd83a769c67a7cf6fd7af489340567d616a0153 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:01:21 -0400 Subject: [PATCH 0094/1072] Add multi-workload iteration support Deploy model once, then iterate over workload_keys running warmup/profiler/benchmark/CSV for each. Backward compatible: empty workload_keys falls back to single workload_key. Co-authored-by: Cursor --- projects/rhaiis/orchestration/cli.py | 2 +- .../orchestration/config.d/workloads.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 188 +++++++++++------- projects/rhaiis/orchestration/test_rhaiis.py | 6 +- 5 files changed, 120 insertions(+), 81 deletions(-) diff --git a/projects/rhaiis/orchestration/cli.py b/projects/rhaiis/orchestration/cli.py index a6cc2e626..b65a07c54 100644 --- a/projects/rhaiis/orchestration/cli.py +++ b/projects/rhaiis/orchestration/cli.py @@ -113,7 +113,7 @@ def test( try: test_phase.run( model_key=model_key, - workload_key=workload_key, + workload_keys=[workload_key], namespace=namespace, deployment_name=deployment_name, ) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 113e55d49..0ef8e0079 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -10,8 +10,8 @@ profile1: profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" - rates: [1, 50, 100, 200, 300] - max_seconds: 450 + rates: [1] + max_seconds: 45 profile3: data: "prompt_tokens=2048,output_tokens=128" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 7323c7d0e..f5aa9224c 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -22,6 +22,7 @@ tests: rhaiis: model_key: qwen3-0_6b workload_key: profile1 + workload_keys: [] run_benchmark: true warmup: true version: "" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ef119d5d9..0fbd21e36 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -13,14 +13,14 @@ def run( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: return run_and_postprocess( do_test, model_key=model_key, - workload_key=workload_key, + workload_keys=workload_keys, namespace=namespace, deployment_name=deployment_name, ) @@ -29,14 +29,14 @@ def run( def do_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: with env.NextArtifactDir("testing"): return _run_test( model_key=model_key, - workload_key=workload_key, + workload_keys=workload_keys, namespace=namespace, deployment_name=deployment_name, ) @@ -45,15 +45,11 @@ def do_test( def _run_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: - run_uuid = str(_uuid_mod.uuid4()) - logger.info("Run UUID: %s", run_uuid) - model_cfg = runtime_config.get_model(model_key) - workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator from projects.core.library import config as _cfg @@ -67,21 +63,12 @@ def _run_test( vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload) + first_workload = runtime_config.get_workload(workload_keys[0]) + vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) - rates = workload.get("rates", [1]) - max_seconds = workload.get("max_seconds", 180) - - _create_test_labels(model_key, workload_key, accelerator, vllm_args) - - logger.info( - "Testing model=%s workload=%s accelerator=%s", model_cfg["name"], workload_key, accelerator - ) + _create_test_labels(model_key, workload_keys[0], accelerator, vllm_args) - from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( - run as run_guidellm_benchmark, - ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) @@ -120,68 +107,118 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - from projects.core.library import config - - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) - run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) - - if profiler_enabled: - logger.info("Skipping warmup -- profiler step provides its own warmup") - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, + logger.info( + "Running %d workload(s): %s", len(workload_keys), workload_keys, + ) + for wl_key in workload_keys: + _run_workload( + model_key=model_key, + workload_key=wl_key, model_cfg=model_cfg, - workload=workload, - workload_key=workload_key, - benchmark_timeout=benchmark_timeout, - ) - elif warmup_enabled: - _run_warmup_step( + accelerator=accelerator, + accelerator_key=accelerator_key, + gpu_type=gpu_type, + vllm_image=vllm_image, + vllm_args=vllm_args, + benchmark_cfg=benchmark_cfg, deployment_name=deployment_name, namespace=namespace, endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, benchmark_timeout=benchmark_timeout, ) + finally: + _capture_and_cleanup(deployment_name, namespace) - main_benchmark_dir = None - if not run_benchmark: - logger.info("run_benchmark=false, skipping main benchmark") - else: - logger.info("Running benchmark at rates=%s", rates) + return 0 - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) +def _run_workload( + *, + model_key: str, + workload_key: str, + model_cfg: dict, + accelerator: str, + accelerator_key: str, + gpu_type: str, + vllm_image: str, + vllm_args: dict, + benchmark_cfg: dict, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_timeout: int, +) -> None: + """Run a single workload: warmup/profiler, benchmark, PSAP/CSV, uploads.""" + run_uuid = str(_uuid_mod.uuid4()) + logger.info("=== Workload %s (UUID: %s) ===", workload_key, run_uuid) - pre_index = env.next_artifact_index() - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}", - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - ) - from pathlib import Path - candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) - if candidates: - main_benchmark_dir = candidates[0] - finally: - _capture_and_cleanup(deployment_name, namespace) + workload = runtime_config.get_workload(workload_key) + rates = workload.get("rates", [1]) + max_seconds = workload.get("max_seconds", 180) + + from projects.core.library import config + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + + if profiler_enabled: + logger.info("Skipping warmup -- profiler step provides its own warmup") + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=workload_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + main_benchmark_dir = None + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + else: + logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) + + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) + + pre_index = env.next_artifact_index() + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=f"guidellm-{deployment_name}-{workload_key}", + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + ) + from pathlib import Path + candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) + if candidates: + main_benchmark_dir = candidates[0] if main_benchmark_dir: try: @@ -202,8 +239,7 @@ def _run_test( except Exception: logger.warning("Predictor log upload failed; continuing", exc_info=True) - profiler_cfg = runtime_config.get_profiler_config() - if profiler_cfg.get("enabled", False): + if profiler_enabled: try: _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: @@ -226,8 +262,6 @@ def _run_test( except Exception: logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) - return 0 - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict diff --git a/projects/rhaiis/orchestration/test_rhaiis.py b/projects/rhaiis/orchestration/test_rhaiis.py index e2a125b89..69be8db73 100644 --- a/projects/rhaiis/orchestration/test_rhaiis.py +++ b/projects/rhaiis/orchestration/test_rhaiis.py @@ -14,8 +14,12 @@ namespace="rhaiis.namespace", ) def test(_cfg): + workload_keys = config.project.get_config("tests.rhaiis.workload_keys", []) + if not workload_keys: + workload_keys = [_cfg.workload_key] + test_phase.run( model_key=_cfg.model_key, - workload_key=_cfg.workload_key, + workload_keys=workload_keys, namespace=_cfg.namespace, ) From 8773870cfc758925773c9b60ebea134336710a19 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:11:40 -0400 Subject: [PATCH 0095/1072] Restructure multi-workload into two phases Phase 1: warmup/profiler for all workloads, then upload traces Phase 2: benchmark + PSAP/CSV/log for each workload MLflow metadata set once after all workloads complete. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 118 +++++++++++--------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0fbd21e36..31c746c96 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -107,11 +107,52 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + logger.info( "Running %d workload(s): %s", len(workload_keys), workload_keys, ) + + # Phase 1: warmup or profiler for ALL workloads first + for wl_key in workload_keys: + workload = runtime_config.get_workload(wl_key) + if profiler_enabled: + logger.info("Running profiler for workload=%s", wl_key) + _run_profiler_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + workload_key=wl_key, + benchmark_timeout=benchmark_timeout, + ) + elif warmup_enabled: + logger.info("Running warmup for workload=%s", wl_key) + _run_warmup_step( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=workload, + benchmark_timeout=benchmark_timeout, + ) + + if profiler_enabled: + try: + _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) + except Exception: + logger.warning("Profiler trace upload failed; continuing", exc_info=True) + + # Phase 2: benchmark + post-processing for ALL workloads for wl_key in workload_keys: - _run_workload( + _run_workload_benchmark( model_key=model_key, workload_key=wl_key, model_cfg=model_cfg, @@ -126,13 +167,33 @@ def _run_test( endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, ) + + try: + first_workload = runtime_config.get_workload(workload_keys[0]) + first_rates = first_workload.get("rates", [1]) + first_max_seconds = first_workload.get("max_seconds", 180) + _set_mlflow_metadata( + model_key, + ",".join(workload_keys), + model_cfg, + accelerator, + vllm_image, + vllm_args, + benchmark_cfg, + first_rates, + first_max_seconds, + namespace, + deployment_name, + ) + except Exception: + logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) finally: _capture_and_cleanup(deployment_name, namespace) return 0 -def _run_workload( +def _run_workload_benchmark( *, model_key: str, workload_key: str, @@ -148,9 +209,9 @@ def _run_workload( endpoint_url: str, benchmark_timeout: int, ) -> None: - """Run a single workload: warmup/profiler, benchmark, PSAP/CSV, uploads.""" + """Run benchmark and post-processing for a single workload.""" run_uuid = str(_uuid_mod.uuid4()) - logger.info("=== Workload %s (UUID: %s) ===", workload_key, run_uuid) + logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) rates = workload.get("rates", [1]) @@ -161,33 +222,7 @@ def _run_workload( run as run_guidellm_benchmark, ) - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) - - if profiler_enabled: - logger.info("Skipping warmup -- profiler step provides its own warmup") - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=workload_key, - benchmark_timeout=benchmark_timeout, - ) - elif warmup_enabled: - _run_warmup_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - benchmark_timeout=benchmark_timeout, - ) main_benchmark_dir = None if not run_benchmark: @@ -239,29 +274,6 @@ def _run_workload( except Exception: logger.warning("Predictor log upload failed; continuing", exc_info=True) - if profiler_enabled: - try: - _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) - except Exception: - logger.warning("Profiler trace upload failed; continuing", exc_info=True) - - try: - _set_mlflow_metadata( - model_key, - workload_key, - model_cfg, - accelerator, - vllm_image, - vllm_args, - benchmark_cfg, - rates, - max_seconds, - namespace, - deployment_name, - ) - except Exception: - logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict From 2d5256b88188416029ecebfa695afaefb726ee35 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:31:19 -0400 Subject: [PATCH 0096/1072] Include workload key in warmup and profiler pod names guidellm-warmup-{model}-{workload_key} guidellm-profiler-{model}-{workload_key}-{gate_value} Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 31c746c96..6a405205a 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -141,6 +141,7 @@ def _run_test( benchmark_cfg=benchmark_cfg, model_cfg=model_cfg, workload=workload, + workload_key=wl_key, benchmark_timeout=benchmark_timeout, ) @@ -530,6 +531,7 @@ def _run_warmup_step( benchmark_cfg: dict, model_cfg: dict, workload: dict, + workload_key: str, benchmark_timeout: int, ) -> None: """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" @@ -555,7 +557,7 @@ def _run_warmup_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-warmup-{deployment_name}", + name=f"guidellm-warmup-{deployment_name}-{workload_key}", namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, @@ -619,7 +621,7 @@ def _run_profiler_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-profiler-{deployment_name}-{gate_value}", + name=f"guidellm-profiler-{deployment_name}-{workload_key}-{gate_value}", namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, From 4ae0e91c57361c176b94e29899660266bcaef79f Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 13 Jul 2026 16:39:38 -0400 Subject: [PATCH 0097/1072] Add HF_TOKEN to GuideLLM job template Pulls from storage-config secret (optional) to avoid HuggingFace rate limits when downloading tokenizers. Co-authored-by: Cursor --- .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8324631c4..7a74cdd08 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -26,6 +26,12 @@ spec: env: - name: USER value: guidellm + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: storage-config + key: HF_TOKEN + optional: true volumeMounts: - name: home mountPath: /home/guidellm From cbd0f6e2a0b1e32be6d9ca22acda28502d40b03c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:30:42 -0400 Subject: [PATCH 0098/1072] Use prefix-workload-model naming for GuideLLM jobs Format: guidellm-{warmup|bench|profiler}-{workload_key}-{model} Trims model name to fit K8s 63-char limit while keeping the workload key fully visible. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 6a405205a..804f61ba4 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -9,6 +9,16 @@ logger = logging.getLogger(__name__) +_K8S_NAME_MAX = 63 + + +def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: + """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" + base = f"{prefix}-{workload_key}-" + available = _K8S_NAME_MAX - len(base) + model = deployment_name[:available] if available > 0 else "" + return f"{base}{model}".rstrip("-") + def run( *, @@ -244,7 +254,7 @@ def _run_workload_benchmark( pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}-{workload_key}", + name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), namespace=namespace, image=benchmark_image, timeout=benchmark_timeout, @@ -557,7 +567,7 @@ def _run_warmup_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-warmup-{deployment_name}-{workload_key}", + name=_guidellm_job_name("guidellm-warmup", workload_key, deployment_name), namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, @@ -621,7 +631,7 @@ def _run_profiler_step( try: run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-profiler-{deployment_name}-{workload_key}-{gate_value}", + name=_guidellm_job_name("guidellm-profiler", workload_key, deployment_name), namespace=namespace, image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), timeout=benchmark_timeout, From ef3468841fec88b4cadf20f7f88d5f8e1dce32f6 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:49:26 -0400 Subject: [PATCH 0099/1072] Share a single UUID across all workloads in a multi-workload run Generate the UUID once in _run_test and pass it to each _run_workload_benchmark call so all workloads in the same job share the same identifier in CSV, PSAP payload, S3 logs, and regression notifications. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 804f61ba4..940371512 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -85,6 +85,9 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready + run_uuid = str(_uuid_mod.uuid4()) + logger.info("Run UUID for this job: %s", run_uuid) + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -177,6 +180,7 @@ def _run_test( namespace=namespace, endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, + run_uuid=run_uuid, ) try: @@ -219,9 +223,9 @@ def _run_workload_benchmark( namespace: str, endpoint_url: str, benchmark_timeout: int, + run_uuid: str, ) -> None: """Run benchmark and post-processing for a single workload.""" - run_uuid = str(_uuid_mod.uuid4()) logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) From bbc54d39dc85256e6cd36e5a17a1332eb27333e7 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 10:54:47 -0400 Subject: [PATCH 0100/1072] Fix predictor log upload ordering _upload_predictor_log was called inside _run_workload_benchmark, before _capture_and_cleanup had run, so the log file didn't exist yet. Move the upload to _run_test, after capture_isvc_state creates the log file. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 940371512..0e9e2f7de 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -205,6 +205,11 @@ def _run_test( finally: _capture_and_cleanup(deployment_name, namespace) + try: + _upload_predictor_log(run_uuid) + except Exception: + logger.warning("Predictor log upload failed; continuing", exc_info=True) + return 0 @@ -284,11 +289,6 @@ def _run_workload_benchmark( except Exception: logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) - try: - _upload_predictor_log(run_uuid) - except Exception: - logger.warning("Predictor log upload failed; continuing", exc_info=True) - def _create_test_labels( model_key: str, workload_key: str, accelerator: str, vllm_args: dict From 0dd147fb36c6df165a1141545ab8fb2385533a48 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 11:47:19 -0400 Subject: [PATCH 0101/1072] Add vllm-profiler/enabled label to InferenceService template The env-injector webhook requires this label on the pod to inject the profiler sitecustomize.py and PYTHONPATH. Without it, the profiler hook never loads and no traces are generated. Co-authored-by: Cursor --- .../deploy_kserve_isvc/templates/inferenceservice.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index baf29eadd..3d04aa276 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -10,6 +10,7 @@ metadata: prometheus.io/port: "8080" labels: opendatahub.io/dashboard: "true" + vllm-profiler/enabled: "true" name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: From 771212cb90a597d18517930b7e668f1f63d28de0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:13:11 -0400 Subject: [PATCH 0102/1072] Fix profiler trace upload path Traces are saved in per-task artifact subdirectories like *__copy_profiler_traces/artifacts/traces/, not under the parent ARTIFACT_DIR. Glob for trace files across all copy_profiler_traces subdirectories and consolidate if needed before uploading. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0e9e2f7de..f7bb2913b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -676,11 +676,20 @@ def _upload_profiler_traces( from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 - traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces" - if not traces_dir.exists() or not any(traces_dir.iterdir()): + trace_files = sorted(Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*")) + if not trace_files: logger.info("No profiler traces to upload") return + traces_dir = trace_files[0].parent + if len(set(f.parent for f in trace_files)) > 1: + traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces_combined" + traces_dir.mkdir(parents=True, exist_ok=True) + for f in trace_files: + import shutil + shutil.copy2(f, traces_dir / f.name) + logger.info("Found %d profiler trace files in %s", len(trace_files), traces_dir) + version = config.project.get_config("tests.rhaiis.version", "") if not version: logger.info("No version configured, skipping profiler trace upload") From 6d8a7c5f505f4a83c041b2a1652d99104b185bbd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:28:39 -0400 Subject: [PATCH 0103/1072] Allow passing run UUID via tests.rhaiis.run_uuid config override If provided, uses the given UUID instead of generating a new one. Enables correlating FournosJob names with their run UUID for traceability across CSV, S3 logs, and pipeline outputs. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f7bb2913b..a1658de6f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -85,7 +85,8 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - run_uuid = str(_uuid_mod.uuid4()) + from projects.core.library import config as _cfg + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) benchmark_timeout = benchmark_cfg.get("timeout", 14400) From bb3b6e7179e85078465861db2c744d0489e4f5bd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:33:16 -0400 Subject: [PATCH 0104/1072] Annotate FournosJob with run UUID for traceability After generating the UUID, annotate the FournosJob resource with rhaiis.run-uuid so the UUID is visible via oc get without needing to dig through pipeline logs. Fully automatic, no user action needed. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index a1658de6f..55880662b 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -89,6 +89,20 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) + import os, subprocess + fjob_name = os.environ.get("FJOB_NAME", "") + fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") + if fjob_name: + try: + subprocess.run( + ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, + f"rhaiis.run-uuid={run_uuid}", "--overwrite"], + check=False, capture_output=True, timeout=10, + ) + logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) + except Exception: + logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) From b08e37888e63ff733f4d5f187fb1810c01894f3a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 13:45:32 -0400 Subject: [PATCH 0105/1072] Fix FournosJob UUID annotation to target management cluster The test pod's KUBECONFIG points to the target cluster (hera), but FournosJob resources live on the management cluster. Strip KUBECONFIG from the env so oc uses in-cluster service account credentials. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 55880662b..88132e33d 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -94,10 +94,11 @@ def _run_test( fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: try: + mgmt_env = {k: v for k, v in os.environ.items() if k != "KUBECONFIG"} subprocess.run( ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, f"rhaiis.run-uuid={run_uuid}", "--overwrite"], - check=False, capture_output=True, timeout=10, + check=False, capture_output=True, timeout=10, env=mgmt_env, ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: From f272a57f749ee492e0b425c3d46dea1aa81b2922 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 14 Jul 2026 19:14:03 -0400 Subject: [PATCH 0106/1072] Fix profiler trace S3 upload when labels are auto-generated When rhaiis.profiler.labels is empty, labels are auto-generated from workload data at runtime but were not passed to the upload function. Infer them from trace filenames (_run{LABEL}_range pattern) so traces are correctly matched and uploaded to S3. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 88132e33d..0331ce572 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -681,6 +681,25 @@ def _derive_profiler_label(workload: dict) -> str: return f"isl{isl}_osl{osl}" +def _infer_profile_labels_from_traces(trace_files: list) -> list[str]: + """Extract unique profile labels from trace filenames. + + Filenames follow the pattern: trace_rank{R}_pid{P}_run{LABEL}_range{S}-{E}.json + The {LABEL} portion is the gate value used during profiling (e.g. 'isl1000_osl1000'). + """ + import re + + pattern = re.compile(r"_run(.+?)_range\d+-\d+") + labels: list[str] = [] + seen: set[str] = set() + for f in trace_files: + m = pattern.search(f.name) + if m and m.group(1) not in seen: + seen.add(m.group(1)) + labels.append(m.group(1)) + return labels + + def _upload_profiler_traces( model_cfg: dict, accelerator: str, @@ -711,13 +730,18 @@ def _upload_profiler_traces( logger.info("No version configured, skipping profiler trace upload") return + profile_labels = profiler_cfg.get("labels", []) + if not profile_labels: + profile_labels = _infer_profile_labels_from_traces(trace_files) + logger.info("Inferred profile labels from trace filenames: %s", profile_labels) + result = upload_profiler_traces_to_s3( traces_dir, model_name=model_cfg.get("hf_model_id", ""), accelerator=accelerator, tp_size=int(vllm_args.get("tensor-parallel-size", 1)), version=version, - profile_labels=profiler_cfg.get("labels", []), + profile_labels=profile_labels, s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), From 9ecf78fd945aba3e70babfe7036ae5968db69272 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 12:57:18 -0400 Subject: [PATCH 0107/1072] Add profile4 workload (ISL=8000, OSL=1000) Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 0ef8e0079..be6c74c0a 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -20,8 +20,7 @@ profile3: profile4: data: "prompt_tokens=8000,output_tokens=1000" - rates: [1, 25, 50, 75, 100] - max_seconds: 450 + rates: [1] + max_seconds: 45 samples: 50 - vllm_args: - max-model-len: 10000 + From 2c101a91db99980582fbfafac86966255fae7e57 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:14:36 -0400 Subject: [PATCH 0108/1072] Increase wait_copy_pod_ready timeout to 5 minutes IBM VPC block storage volume re-attachment can take 3+ minutes. The previous 2-minute window (24x5s) was insufficient. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 91cbcc445..5a8253abf 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -264,7 +264,7 @@ def create_copy_pod(args, ctx): return f"Created copy pod {ctx.benchmark_name}-copy" -@retry(attempts=24, delay=5, backoff=1.0) +@retry(attempts=60, delay=5, backoff=1.0) @task def wait_copy_pod_ready(args, ctx): """Wait for copy pod to be ready""" From 28193bf4d2a4c2afbc4f1c2c8b48fd5b50829eef Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:30:16 -0400 Subject: [PATCH 0109/1072] Set USER env var in vLLM container to fix getpwuid crash on OpenShift OpenShift assigns arbitrary UIDs without /etc/passwd entries, causing PyTorch's getpass.getuser() -> pwd.getpwuid() to raise KeyError. Setting USER bypasses the pwd lookup entirely. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index baabd0852..42d53cd73 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -43,6 +43,8 @@ spec: {%- endif %} {%- endfor %} env: + - name: USER + value: "vllm" {%- if args.storage_source == "hf" %} - name: HF_HUB_OFFLINE value: "0" From b6d8c06712777207193548c6f0798dde5b63add9 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 13:50:43 -0400 Subject: [PATCH 0110/1072] Add ISVC/SR cleanup to post_cleanup hook Ensures InferenceService and ServingRuntime are deleted during Tekton finally step, covering cases where the test pod is killed before its own try/finally cleanup runs. Co-authored-by: Cursor --- projects/rhaiis/orchestration/prepare_rhaiis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index aa8c3b06a..5f7421035 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -48,6 +48,8 @@ def cleanup(): ns = runtime_config.get_namespace() logger.info(f"Cleaning up rhaiis benchmark resources in {ns}") + oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) logger.info("Cleanup complete") From b9e23804fc510411111b0efb72e3b1fcc1918b78 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 16:39:55 -0400 Subject: [PATCH 0111/1072] Update profile1 workload: rate=50, max_seconds=150 Needed to trigger regression comparison (concurrency=1 is excluded). Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index be6c74c0a..c413b505e 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,8 +5,8 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [1] - max_seconds: 45 + rates: [50] + max_seconds: 150 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" From f53d950c66e00d0345ed902b36a614beff0ff80b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 20:52:55 -0400 Subject: [PATCH 0112/1072] Add AI agent analysis integration for regression reports Port Model Furnace agent analysis to Forge: request AI analysis when severe regressions are detected, generate HTML report, upload to S3, and include report link in Slack notifications. Configure agent gateway URL for in-cluster access from management cluster. Co-authored-by: Cursor --- projects/core/notifications/send.py | 5 + .../rhaiis/orchestration/config.d/rhaiis.yaml | 5 + projects/rhaiis/orchestration/test_phase.py | 107 ++++++ projects/rhaiis/postprocess/agent.py | 306 ++++++++++++++++++ 4 files changed, 423 insertions(+) create mode 100644 projects/rhaiis/postprocess/agent.py diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 10415559f..0e96228c5 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -755,6 +755,7 @@ def send_regression_notification( webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, + report_url: str = "", ) -> bool: """Send regression/improvement Slack notification from analysis results. @@ -770,6 +771,7 @@ def send_regression_notification( webhook_vault: Vault containing slack-webhook-url file notification_vault: Fallback vault for Bot-Token Slack credentials dry_run: Log only, don't send + report_url: Optional presigned URL to an agent analysis report Returns: True if notification sent successfully @@ -834,6 +836,8 @@ def send_regression_notification( else: user_line = "" + report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + message = ( f"{icon} *{headline}*\n" f"{user_line}" @@ -841,6 +845,7 @@ def send_regression_notification( f"*Model:* {model}\n" f"*Accelerator:* {accelerator}\n" f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"{report_line}" f"*Changes:*\n{details}" ) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 96f1d7def..545ad6076 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -48,3 +48,8 @@ profiler: labels: [] rates: [200] max_seconds: 200 + +agent_analysis: + enabled: false + severity_threshold: 10 + url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0331ce572..5f0894ccd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -509,6 +509,15 @@ def _run_regression_check( ) if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + report_url = "" + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if agent_cfg.get("enabled", False): + report_url = _run_agent_analysis( + analysis, model_cfg, accelerator, + current_version, compare_version, run_uuid, + severity_threshold=agent_cfg.get("severity_threshold", 10), + ) + from projects.core.notifications.send import send_regression_notification send_regression_notification( @@ -519,6 +528,7 @@ def _run_regression_check( slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", + report_url=report_url, ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) @@ -528,6 +538,103 @@ def _run_regression_check( os.unlink(consolidated_path) +def _run_agent_analysis( + analysis: dict, + model_cfg: dict, + accelerator: str, + current_version: str, + compare_version: str, + run_uuid: str, + *, + severity_threshold: int = 10, +) -> str: + """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.agent import ( + AGENT_SEVERITY_THRESHOLD, + build_pr_followup_prompt, + check_agent_connectivity, + markdown_to_html, + request_agent_analysis, + send_followup, + ) + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + agent_url = agent_cfg.get("url", "") + if not agent_url: + logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") + return "" + + threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD + severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] + if not severe: + logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) + return "" + + ok, detail = check_agent_connectivity(agent_url) + if not ok: + logger.warning("Agent not reachable, skipping analysis: %s", detail) + return "" + + tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + model = model_cfg.get("hf_model_id", "") + improvements = analysis.get("improvements", []) + + agent_response = request_agent_analysis( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe, + job_id=run_uuid, + improvements=improvements if improvements else None, + agent_url=agent_url, + ) + if not agent_response: + return "" + + pr_prompt = build_pr_followup_prompt(current_version, compare_version) + pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) + if pr_analysis: + agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" + + html_content = markdown_to_html( + agent_response, run_uuid, model, current_version, compare_version, + ) + html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" + html_path.write_text(html_content, encoding="utf-8") + logger.info("Agent analysis saved to %s", html_path) + + try: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if credentials_path: + s3 = create_s3_client(credentials_path) + s3_bucket = "psap-dashboard-data" + s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" + s3.upload_file( + str(html_path), s3_bucket, s3_key, + ExtraArgs={"ContentType": "text/html"}, + ) + report_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": s3_bucket, "Key": s3_key}, + ExpiresIn=2592000, + ) + logger.info("Agent analysis uploaded to S3, presigned URL generated") + return report_url + except Exception: + logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) + + return "" + + def _upload_predictor_log(run_uuid: str) -> None: """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" from pathlib import Path diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py new file mode 100644 index 000000000..22732edc7 --- /dev/null +++ b/projects/rhaiis/postprocess/agent.py @@ -0,0 +1,306 @@ +"""Client for the PSAP AI agent's /v1/stream endpoint. + +Sends regression context as a natural-language prompt and collects the +streamed AI response into a markdown string. Ported from model-furnace. +""" + +import json +import logging +import uuid +from typing import Optional +from urllib.error import URLError +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +DEFAULT_AGENT_URL = ( + "https://psap-agent-staging-psap-ai-agent.apps.ocp4.intlab.redhat.com/v1/stream" +) +AGENT_TIMEOUT_SECONDS = 600 +AGENT_HEALTH_TIMEOUT = 15 +AGENT_SEVERITY_THRESHOLD = 10 + + +def check_agent_connectivity(agent_url: str = "") -> tuple[bool, str]: + """Verify that the agent endpoint is reachable. + + Tries a lightweight GET to the agent health URL. Returns (ok, detail). + """ + url = agent_url or DEFAULT_AGENT_URL + base_url = url.rsplit("/", 2)[0] + health_url = f"{base_url}/health" + + try: + req = Request(health_url, method="GET") + resp = urlopen(req, timeout=AGENT_HEALTH_TIMEOUT) # noqa: S310 + status = resp.getcode() + if status and status < 400: + return True, f"Agent reachable at {health_url} (HTTP {status})" + return False, f"Agent returned HTTP {status} at {health_url}" + except (URLError, OSError) as e: + return False, f"Cannot reach agent at {health_url}: {e}" + except Exception as e: + return False, f"Unexpected error checking agent at {health_url}: {e}" + + +def _build_prompt( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + improvements: Optional[list] = None, +) -> str: + """Build a natural-language prompt for the agent from regression context.""" + lines = [ + f"Analyze the performance regression between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}.", + "", + "The following metrics regressed significantly (>10%):", + ] + for r in severe_regressions: + direction = "dropped" if r["pct_diff"] < 0 else "increased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + if improvements: + lines.append("") + lines.append("The following metrics improved significantly:") + for r in improvements: + direction = "increased" if r["pct_diff"] > 0 else "decreased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + lines.append("") + lines.append( + f"Check if the runtime args differ between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}. " + ) + lines.append("") + lines.append( + "Provide root cause analysis using pytorch profiler traces, vLLM logs, " + "and vLLM source code where available." + ) + return "\n".join(lines) + + +def build_pr_followup_prompt( + current_version: str, + compare_version: str, +) -> str: + """Build a followup prompt asking the agent to identify relevant PRs.""" + return ( + "Based on your previous analysis, use the compare_vllm_versions and " + "get_vllm_pull_request tools to identify which specific pull requests " + f"between vLLM {compare_version} and {current_version} most likely " + "explain the performance differences you found. For each PR you " + "identify, include its GitHub link and a brief explanation of why " + "it's relevant to the regressions or improvements you observed." + ) + + +def request_agent_analysis( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + job_id: str = "", + improvements: Optional[list] = None, + agent_url: str = "", +) -> Optional[str]: + """Send regression context to the PSAP agent and return its analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url or DEFAULT_AGENT_URL + + prompt = _build_prompt( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe_regressions, + improvements=improvements, + ) + + session_key = f"forge-rhaiis-{job_id or uuid.uuid4()}" + body = { + "message": prompt, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Requesting agent analysis for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent analysis received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent analysis for job %s: %s", job_id, e) + return None + + +def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[str]: + """Send a followup message to the agent on an existing session. + + Reuses the same thread_id/session_id as request_agent_analysis so the + agent retains full conversation context from the initial analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url or DEFAULT_AGENT_URL + + session_key = f"forge-rhaiis-{job_id}" + body = { + "message": message, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Sending agent followup for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent followup received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty followup response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent followup request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent followup for job %s: %s", job_id, e) + return None + + +def _collect_response(resp) -> Optional[str]: + """Read the NDJSON stream and extract the final AI message content.""" + collected_tokens = [] + final_message = None + + for raw_line in resp: + line = raw_line.decode("utf-8", errors="replace").strip() + if not line or line == "[DONE]": + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + event_type = event.get("type") + content = event.get("content") + + if event_type == "token" and isinstance(content, str): + collected_tokens.append(content) + elif ( + event_type == "message" + and isinstance(content, dict) + and content.get("type") == "ai" + and content.get("content") + ): + final_message = content["content"] + + if final_message: + return final_message + if collected_tokens: + return "".join(collected_tokens) + return None + + +def markdown_to_html( + md_text: str, + job_id: str, + model: str, + current_version: str, + compare_version: str, +) -> str: + """Convert markdown analysis to a self-contained HTML page.""" + try: + import markdown as md_lib + + body = md_lib.markdown(md_text, extensions=["tables", "fenced_code"]) + except ImportError: + import html as html_lib + + body = f"
{html_lib.escape(md_text)}
" + + from datetime import datetime, timezone + + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + return f""" + + + + +Analysis — {job_id} + + + +

Regression Analysis Report

+
+ Job: {job_id} + Model: {model} + Versions: {current_version} vs {compare_version} + Generated: {timestamp} +
+{body} + +""" From d91ae9158603be700c05bdefc3800fbe11d40898 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 21:02:40 -0400 Subject: [PATCH 0113/1072] Add standalone agent analysis mode (no benchmark required) When run_benchmark=false but agent_analysis.enabled=true, download existing data from the S3 consolidated CSV and run regression check + agent analysis against historical data. This allows testing the agent analysis pipeline without running a new benchmark. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 89 +++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 5f0894ccd..1de90d513 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -263,6 +263,10 @@ def _run_workload_benchmark( main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") + try: + _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) else: logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) @@ -462,6 +466,91 @@ def _generate_and_sync_dashboard_csv( _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) +def _run_standalone_analysis( + model_cfg: dict, + accelerator: str, + vllm_args: dict, + *, + run_uuid: str = "", +) -> None: + """Run regression check + agent analysis using existing S3 data (no new benchmark).""" + import tempfile + from pathlib import Path + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if not agent_cfg.get("enabled", False): + logger.info("Standalone analysis skipped: agent_analysis not enabled") + return + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not version or not compare_version: + logger.info("Standalone analysis skipped: version or compare_version not configured") + return + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") + s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") + vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + + credentials_path = get_aws_credentials(vault_name, "aws.credentials") + if not credentials_path: + logger.warning("AWS credentials not available, skipping standalone analysis") + return + + consolidated_path = None + current_csv_path = None + try: + import pandas as pd + + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + + df = pd.read_csv(consolidated_path, on_bad_lines="warn") + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].str.strip() + + model_id = model_cfg.get("hf_model_id", "") + tp = str(vllm_args.get("tensor-parallel-size", 1)) + + current_rows = df[ + (df["version"] == version) + & (df["model"] == model_id) + & (df["accelerator"] == accelerator) + & (df["TP"].astype(str) == tp) + ] + + if current_rows.empty: + logger.warning( + "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", + version, model_id, accelerator, tp, + ) + return + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + current_csv_path = tmp.name + current_rows.to_csv(tmp, index=False) + + logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) + _run_regression_check( + current_csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + finally: + import os + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + if current_csv_path and os.path.exists(current_csv_path): + os.unlink(current_csv_path) + + def _run_regression_check( csv_path, compare_version: str, From 9b7a673c6cc272185b514276f68c31975ae87d79 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:06:54 -0400 Subject: [PATCH 0114/1072] Skip warmup when run_benchmark and profiler are both disabled Warmup only makes sense as a pre-benchmark step. When both run_benchmark=false and profiler.enabled=false, skip warmup regardless of its own setting to avoid deploying a model needlessly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 1de90d513..4fc6211fd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -140,7 +140,12 @@ def _run_test( profiler_cfg = runtime_config.get_profiler_config() profiler_enabled = profiler_cfg.get("enabled", False) - warmup_enabled = config.project.get_config("tests.rhaiis.warmup", True) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + warmup_enabled = ( + config.project.get_config("tests.rhaiis.warmup", True) + and run_benchmark + and not profiler_enabled + ) logger.info( "Running %d workload(s): %s", len(workload_keys), workload_keys, From 3cabbb44276f3c984a382792c49c2dd55b7fe1bd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:24:38 -0400 Subject: [PATCH 0115/1072] Use chip family (H200) not accelerator_key (H200_HERA2) for CSV filtering The cluster_tag suffix is only used in the CSV run ID column. The accelerator column stores the chip family (e.g. H200). Strip the cluster_tag when filtering S3 data and when passing to regression check / notifications. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 4fc6211fd..697c317ef 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -418,7 +418,7 @@ def _generate_psap_payload( def _generate_and_sync_dashboard_csv( model_cfg: dict, - accelerator: str, + accelerator_key: str, workload_key: str, vllm_args: dict, *, @@ -429,6 +429,9 @@ def _generate_and_sync_dashboard_csv( from projects.core.library import config from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv + # Use chip family only (e.g. "H200") — cluster_tag is only for run ID in the CSV + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) if not psap_files: logger.warning("No PSAP JSON files found, skipping dashboard CSV") @@ -473,7 +476,7 @@ def _generate_and_sync_dashboard_csv( def _run_standalone_analysis( model_cfg: dict, - accelerator: str, + accelerator_key: str, vllm_args: dict, *, run_uuid: str = "", @@ -506,6 +509,9 @@ def _run_standalone_analysis( logger.warning("AWS credentials not available, skipping standalone analysis") return + # CSV stores just the chip family (e.g. "H200"), not the full accelerator_key with cluster tag + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + consolidated_path = None current_csv_path = None try: From 6827edf4c5037fe89ba22eaef6878d505699df18 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 15 Jul 2026 23:50:30 -0400 Subject: [PATCH 0116/1072] Rename gpu_types key 'amd' to 'amd_mi300x' for clarity Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 545ad6076..66df54d5c 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -4,7 +4,7 @@ cluster_tag: "" gpu_types: nvidia: h200 - amd: mi300x + amd_mi300x: mi300x l40s: l40s images: From 813a0989b02578ca2da7c952d8ed054dd6a9ad5d Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:06:33 -0400 Subject: [PATCH 0117/1072] Fix TP column comparison: CSV stores floats (2.0), convert to int before string match Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 697c317ef..427241d60 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -534,7 +534,7 @@ def _run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].astype(str) == tp) + & (df["TP"].astype(float).astype(int).astype(str) == tp) ] if current_rows.empty: From 09e2bc9674d960b260d33d20dc9c8c7c74f395ab Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:24:38 -0400 Subject: [PATCH 0118/1072] Skip vLLM deployment when only standalone analysis is needed When both run_benchmark=false and profiler.enabled=false, the pipeline only needs S3 data for regression analysis. Skip the InferenceService deployment entirely and run standalone analysis directly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 427241d60..7cd267030 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -104,6 +104,22 @@ def _run_test( except Exception: logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + + # Standalone analysis only — no deployment needed + if not run_benchmark and not profiler_enabled: + logger.info("run_benchmark=false and profiler=false, running standalone analysis only") + for wl_key in workload_keys: + try: + _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) + return 0 + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -136,11 +152,6 @@ def _run_test( endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - from projects.core.library import config - - profiler_cfg = runtime_config.get_profiler_config() - profiler_enabled = profiler_cfg.get("enabled", False) - run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) warmup_enabled = ( config.project.get_config("tests.rhaiis.warmup", True) and run_benchmark From cf3ba112c2ae4a489b743898ab2044118785343b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 00:30:50 -0400 Subject: [PATCH 0119/1072] Handle NaN values in TP column during CSV filtering The TP column can have NaN for rows with missing data. Fill with -1 before int conversion so those rows are excluded from the filter. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 7cd267030..356beba99 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -545,7 +545,7 @@ def _run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].astype(float).astype(int).astype(str) == tp) + & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) ] if current_rows.empty: From 2fd8142527189cb2e89d76d34973410817845876 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 16 Jul 2026 13:20:47 -0400 Subject: [PATCH 0120/1072] Fix workload_keys filtering, add TP/dashboard to Slack notifications MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire restrict_profiles through standalone analysis → regression check → compare_runs so workload_keys actually filters which profiles appear in Slack notifications (previously all S3 profiles were reported) - Remove redundant per-key loop in standalone analysis path - Add TP/DP parallelism info to Slack notification message - Add pre-filtered RHAIIS dashboard link to Slack notification - Rename "Model Furnace" references to "Fournos" in message text Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 2 + projects/core/notifications/send.py | 73 +++++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 24 +++++-- 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index f2ef0b4bd..78f33839b 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -178,6 +178,7 @@ def run_regression_analysis( compare_version: str, current_version: str, output_file: Path, + restrict_profiles: Optional[list[str]] = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. @@ -189,6 +190,7 @@ def run_regression_analysis( str(consolidated_csv_path), compare_version, current_version, + restrict_profiles=restrict_profiles, ) if skip_reason: diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 0e96228c5..babe3b77d 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -745,6 +745,54 @@ def _get_webhook_url(vault_name: str) -> str | None: return path.read_text().strip() +DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +PROFILE_DISPLAY_NAMES = { + "profile1": "Profile A: Balanced (1k/1k)", + "profile2": "Profile B: Variable Workload (512/2k)", + "profile3": "Profile C: Large Prompt (2k/128)", + "profile4": "Profile D: Prefill Heavy (8k/1k)", +} + + +def _build_dashboard_url( + *, + model: str = "", + accelerator: str = "", + current_version: str = "", + compare_version: str = "", + profiles: list[str] | None = None, + tp: str = "", +) -> str: + """Build a RHAIIS dashboard URL with filters pre-selected.""" + from urllib.parse import quote, urlencode + + params: dict[str, str] = {"view": "RHAIIS Dashboard"} + + if accelerator: + params["accelerators"] = accelerator + if model: + params["models"] = model + + versions = ",".join(v for v in [current_version, compare_version] if v) + if versions: + params["versions"] = versions + + if profiles: + display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] + params["profile"] = ",".join(display_names) + + if tp: + try: + params["tp_sizes"] = f"{float(tp):.1f}" + except (ValueError, TypeError): + params["tp_sizes"] = tp + + params["section"] = "performance_plots" + + return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" + + def send_regression_notification( analysis_result: dict, *, @@ -756,6 +804,8 @@ def send_regression_notification( notification_vault: str | None = None, dry_run: bool = False, report_url: str = "", + tp: str = "", + dp: str = "", ) -> bool: """Send regression/improvement Slack notification from analysis results. @@ -772,6 +822,8 @@ def send_regression_notification( notification_vault: Fallback vault for Bot-Token Slack credentials dry_run: Log only, don't send report_url: Optional presigned URL to an agent analysis report + tp: Tensor parallelism size for display + dp: Data parallelism size for display Returns: True if notification sent successfully @@ -838,14 +890,35 @@ def send_regression_notification( report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + parallelism_line = "" + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + if parallelism_parts: + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" + + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + profiles=profiles if profiles else None, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + message = ( f"{icon} *{headline}*\n" f"{user_line}" f"*Job:* `{job_id}`\n" f"*Model:* {model}\n" f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" f"*Versions:* {current_version} vs {compare_version} (baseline)\n" f"{report_line}" + f"{dashboard_line}" f"*Changes:*\n{details}" ) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 356beba99..ea9e63dc5 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -113,11 +113,13 @@ def _run_test( # Standalone analysis only — no deployment needed if not run_benchmark and not profiler_enabled: logger.info("run_benchmark=false and profiler=false, running standalone analysis only") - for wl_key in workload_keys: - try: - _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) + try: + _run_standalone_analysis( + model_cfg, accelerator_key, vllm_args, + run_uuid=run_uuid, restrict_profiles=workload_keys or None, + ) + except Exception: + logger.warning("Standalone analysis failed; continuing", exc_info=True) return 0 benchmark_timeout = benchmark_cfg.get("timeout", 14400) @@ -482,7 +484,7 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid) + _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) def _run_standalone_analysis( @@ -491,6 +493,7 @@ def _run_standalone_analysis( vllm_args: dict, *, run_uuid: str = "", + restrict_profiles: list[str] | None = None, ) -> None: """Run regression check + agent analysis using existing S3 data (no new benchmark).""" import tempfile @@ -561,7 +564,8 @@ def _run_standalone_analysis( logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) _run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, + current_csv_path, compare_version, version, model_cfg, accelerator, + run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -581,6 +585,8 @@ def _run_regression_check( accelerator: str, *, run_uuid: str = "", + restrict_profiles: list[str] | None = None, + vllm_args: dict | None = None, ) -> None: import tempfile from pathlib import Path @@ -617,6 +623,7 @@ def _run_regression_check( compare_version=compare_version, current_version=current_version, output_file=output_file, + restrict_profiles=restrict_profiles, ) if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: @@ -631,6 +638,7 @@ def _run_regression_check( from projects.core.notifications.send import send_regression_notification + _vllm = vllm_args or {} send_regression_notification( analysis, model=model_cfg.get("hf_model_id", ""), @@ -640,6 +648,8 @@ def _run_regression_check( webhook_vault="psap-forge-rhaiis-slack", notification_vault="psap-forge-notifications", report_url=report_url, + tp=str(_vllm.get("tensor-parallel-size", "")), + dp=str(_vllm.get("data-parallel-size", "")), ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) From b244b49436d0e46e84475295aed66822584111a0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 17 Jul 2026 00:13:23 -0400 Subject: [PATCH 0121/1072] Address PR review: move RHAIIS-specific code out of core/caliper - Move PROFILE_MAP, METRICS from caliper engine to rhaiis postprocess - Make compare_runs/run_regression_analysis accept profile_map and metrics as parameters instead of hardcoded constants - Move send_regression_notification, dashboard URL builder, and profile display names from core/notifications to rhaiis postprocess - Consolidate Slack webhook into psap-forge-notifications vault, remove separate psap-forge-rhaiis-slack vault - Remove hardcoded DEFAULT_AGENT_URL from agent.py, require caller to provide URL from config - Add docstring to enable_profiler_gate run function - Clean up vault descriptions to be project-generic Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 29 +-- projects/core/notifications/send.py | 198 ---------------- .../rhaiis/orchestration/config.d/rhaiis.yaml | 4 +- projects/rhaiis/orchestration/config.yaml | 1 - projects/rhaiis/orchestration/test_phase.py | 7 +- projects/rhaiis/postprocess/agent.py | 15 +- projects/rhaiis/postprocess/regression.py | 224 ++++++++++++++++++ .../toolbox/enable_profiler_gate/main.py | 11 + vaults/psap-forge-dashboard-s3.yaml | 7 +- vaults/psap-forge-notifications.yaml | 5 + vaults/psap-forge-rhaiis-slack.yaml | 10 - 11 files changed, 268 insertions(+), 243 deletions(-) create mode 100644 projects/rhaiis/postprocess/regression.py delete mode 100644 vaults/psap-forge-rhaiis-slack.yaml diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 78f33839b..847fbf159 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -15,21 +15,6 @@ logger = logging.getLogger(__name__) -PROFILE_MAP = { - (1000, 1000): "profile1", - (512, 2048): "profile2", - (2048, 128): "profile3", - (8000, 1000): "profile4", -} - -METRICS = { - "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, - "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, - "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, - "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, - "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, -} - @dataclass class RegressionResult: @@ -55,10 +40,16 @@ def compare_runs( consolidated_csv_path: str, compare_version: str, current_version: str, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], restrict_profiles: Optional[list[str]] = None, ) -> tuple[list[RegressionResult], str]: """Compare current run metrics against a baseline version. + Args: + profile_map: Mapping of (input_tokens, output_tokens) -> profile name. + metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. """ import pandas as pd @@ -89,7 +80,7 @@ def _assign_profile(row): osl = int(float(row["output toks"])) except (ValueError, TypeError): return None - return PROFILE_MAP.get((isl, osl)) + return profile_map.get((isl, osl)) current_df["_profile"] = current_df.apply(_assign_profile, axis=1) baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) @@ -132,7 +123,7 @@ def _assign_profile(row): cur_common = cur[cur["intended concurrency"].isin(common_conc)] base_common = base[base["intended concurrency"].isin(common_conc)] - for col, meta in METRICS.items(): + for col, meta in metrics.items(): if col not in cur_common.columns or col not in base_common.columns: continue @@ -178,6 +169,8 @@ def run_regression_analysis( compare_version: str, current_version: str, output_file: Path, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], restrict_profiles: Optional[list[str]] = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. @@ -190,6 +183,8 @@ def run_regression_analysis( str(consolidated_csv_path), compare_version, current_version, + profile_map=profile_map, + metrics=metrics, restrict_profiles=restrict_profiles, ) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index babe3b77d..dfc494765 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -745,204 +745,6 @@ def _get_webhook_url(vault_name: str) -> str | None: return path.read_text().strip() -DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" - -PROFILE_DISPLAY_NAMES = { - "profile1": "Profile A: Balanced (1k/1k)", - "profile2": "Profile B: Variable Workload (512/2k)", - "profile3": "Profile C: Large Prompt (2k/128)", - "profile4": "Profile D: Prefill Heavy (8k/1k)", -} - - -def _build_dashboard_url( - *, - model: str = "", - accelerator: str = "", - current_version: str = "", - compare_version: str = "", - profiles: list[str] | None = None, - tp: str = "", -) -> str: - """Build a RHAIIS dashboard URL with filters pre-selected.""" - from urllib.parse import quote, urlencode - - params: dict[str, str] = {"view": "RHAIIS Dashboard"} - - if accelerator: - params["accelerators"] = accelerator - if model: - params["models"] = model - - versions = ",".join(v for v in [current_version, compare_version] if v) - if versions: - params["versions"] = versions - - if profiles: - display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] - params["profile"] = ",".join(display_names) - - if tp: - try: - params["tp_sizes"] = f"{float(tp):.1f}" - except (ValueError, TypeError): - params["tp_sizes"] = tp - - params["section"] = "performance_plots" - - return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" - - -def send_regression_notification( - analysis_result: dict, - *, - model: str = "", - accelerator: str = "", - job_id: str = "", - slack_user: str = "", - webhook_vault: str | None = None, - notification_vault: str | None = None, - dry_run: bool = False, - report_url: str = "", - tp: str = "", - dp: str = "", -) -> bool: - """Send regression/improvement Slack notification from analysis results. - - Uses a webhook URL (from *webhook_vault*) when available, falling - back to the Bot-Token path (from *notification_vault*). - - Args: - analysis_result: Output from run_regression_analysis() - model: Model name for display - accelerator: Accelerator name for display - job_id: Job identifier - slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials - dry_run: Log only, don't send - report_url: Optional presigned URL to an agent analysis report - tp: Tensor parallelism size for display - dp: Data parallelism size for display - - Returns: - True if notification sent successfully - """ - status = analysis_result.get("status") - if status == "skipped": - reason = analysis_result.get("reason", "unknown") - logger.info("Regression analysis skipped: %s", reason) - return True - - if status != "completed": - return True - - regressions = analysis_result.get("regressions", []) - improvements = analysis_result.get("improvements", []) - - if not regressions and not improvements: - logger.info("No regressions or improvements to report") - return True - - current_version = analysis_result.get("current_version", "") - compare_version = analysis_result.get("compare_version", "") - - if regressions and improvements: - icon = ":warning:" - headline = "Performance regressions and improvements detected" - elif regressions: - icon = ":warning:" - headline = "Performance regressions detected" - else: - icon = ":large_green_circle:" - headline = "Performance improvements detected" - - detail_lines = [] - all_results = analysis_result.get("all_results", []) - profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) - - for profile in profiles: - profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] - detail_lines.append(f"\n*{profile}:*") - for r in profile_results: - if r.get("is_regression"): - direction = "dropped" if r["pct_diff"] < 0 else "increased" - detail_lines.append( - f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " - f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" - ) - else: - detail_lines.append( - f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " - f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" - ) - - details = "\n".join(detail_lines) - - import re as _re - - if slack_user and _re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" - - report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" - - parallelism_line = "" - parallelism_parts = [] - if tp: - parallelism_parts.append(f"TP={tp}") - if dp: - parallelism_parts.append(f"DP={dp}") - if parallelism_parts: - parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" - - dashboard_url = _build_dashboard_url( - model=model, - accelerator=accelerator, - current_version=current_version, - compare_version=compare_version, - profiles=profiles if profiles else None, - tp=tp, - ) - dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" - - message = ( - f"{icon} *{headline}*\n" - f"{user_line}" - f"*Job:* `{job_id}`\n" - f"*Model:* {model}\n" - f"*Accelerator:* {accelerator}\n" - f"{parallelism_line}" - f"*Versions:* {current_version} vs {compare_version} (baseline)\n" - f"{report_line}" - f"{dashboard_line}" - f"*Changes:*\n{details}" - ) - - if dry_run: - logger.info("DRY RUN regression notification:\n%s", message) - return True - - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) - - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 66df54d5c..12da18a24 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -4,7 +4,7 @@ cluster_tag: "" gpu_types: nvidia: h200 - amd_mi300x: mi300x + amd: mi300x l40s: l40s images: @@ -17,7 +17,7 @@ deploy: memory_request: "16Gi" storage_source: hf storage_pvc: model-pvc - image_pull_secret: npalaska-image-pull + image_pull_secret: "" service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index f5aa9224c..1442b8c3a 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -3,7 +3,6 @@ vaults: - psap-forge-mlflow-export - psap-forge-aws-s3-export - psap-forge-dashboard-s3 -- psap-forge-rhaiis-slack benchmarks: guidellm: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ea9e63dc5..20038e4bd 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -594,6 +594,7 @@ def _run_regression_check( from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials from projects.caliper.engine.kpi.analyze import run_regression_analysis from projects.core.library import config + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") @@ -623,6 +624,8 @@ def _run_regression_check( compare_version=compare_version, current_version=current_version, output_file=output_file, + profile_map=PROFILE_MAP, + metrics=METRICS, restrict_profiles=restrict_profiles, ) @@ -636,7 +639,7 @@ def _run_regression_check( severity_threshold=agent_cfg.get("severity_threshold", 10), ) - from projects.core.notifications.send import send_regression_notification + from projects.rhaiis.postprocess.regression import send_regression_notification _vllm = vllm_args or {} send_regression_notification( @@ -645,7 +648,7 @@ def _run_regression_check( accelerator=accelerator, job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-rhaiis-slack", + webhook_vault="psap-forge-notifications", notification_vault="psap-forge-notifications", report_url=report_url, tp=str(_vllm.get("tensor-parallel-size", "")), diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py index 22732edc7..d08140310 100644 --- a/projects/rhaiis/postprocess/agent.py +++ b/projects/rhaiis/postprocess/agent.py @@ -13,20 +13,17 @@ logger = logging.getLogger(__name__) -DEFAULT_AGENT_URL = ( - "https://psap-agent-staging-psap-ai-agent.apps.ocp4.intlab.redhat.com/v1/stream" -) AGENT_TIMEOUT_SECONDS = 600 AGENT_HEALTH_TIMEOUT = 15 AGENT_SEVERITY_THRESHOLD = 10 -def check_agent_connectivity(agent_url: str = "") -> tuple[bool, str]: +def check_agent_connectivity(agent_url: str) -> tuple[bool, str]: """Verify that the agent endpoint is reachable. Tries a lightweight GET to the agent health URL. Returns (ok, detail). """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url base_url = url.rsplit("/", 2)[0] health_url = f"{base_url}/health" @@ -109,15 +106,15 @@ def request_agent_analysis( compare_version: str, tp: str, severe_regressions: list, + agent_url: str, job_id: str = "", improvements: Optional[list] = None, - agent_url: str = "", ) -> Optional[str]: """Send regression context to the PSAP agent and return its analysis. Returns the agent's response as a markdown string, or None on failure. """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url prompt = _build_prompt( model=model, @@ -168,7 +165,7 @@ def request_agent_analysis( return None -def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[str]: +def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: """Send a followup message to the agent on an existing session. Reuses the same thread_id/session_id as request_agent_analysis so the @@ -176,7 +173,7 @@ def send_followup(message: str, job_id: str, agent_url: str = "") -> Optional[st Returns the agent's response as a markdown string, or None on failure. """ - url = agent_url or DEFAULT_AGENT_URL + url = agent_url session_key = f"forge-rhaiis-{job_id}" body = { diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py new file mode 100644 index 000000000..d992e859d --- /dev/null +++ b/projects/rhaiis/postprocess/regression.py @@ -0,0 +1,224 @@ +"""RHAIIS-specific profile/metric definitions and Slack notification for regression analysis.""" + +from __future__ import annotations + +import logging +import re + +logger = logging.getLogger(__name__) + +PROFILE_MAP = { + (1000, 1000): "profile1", + (512, 2048): "profile2", + (2048, 128): "profile3", + (8000, 1000): "profile4", +} + +METRICS = { + "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, + "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, + "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, + "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, + "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, +} + +DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +PROFILE_DISPLAY_NAMES = { + "profile1": "Profile A: Balanced (1k/1k)", + "profile2": "Profile B: Variable Workload (512/2k)", + "profile3": "Profile C: Large Prompt (2k/128)", + "profile4": "Profile D: Prefill Heavy (8k/1k)", +} + + +def _build_dashboard_url( + *, + model: str = "", + accelerator: str = "", + current_version: str = "", + compare_version: str = "", + profiles: list[str] | None = None, + tp: str = "", +) -> str: + """Build a RHAIIS dashboard URL with filters pre-selected.""" + from urllib.parse import quote, urlencode + + params: dict[str, str] = {"view": "RHAIIS Dashboard"} + + if accelerator: + params["accelerators"] = accelerator + if model: + params["models"] = model + + versions = ",".join(v for v in [current_version, compare_version] if v) + if versions: + params["versions"] = versions + + if profiles: + display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] + params["profile"] = ",".join(display_names) + + if tp: + try: + params["tp_sizes"] = f"{float(tp):.1f}" + except (ValueError, TypeError): + params["tp_sizes"] = tp + + params["section"] = "performance_plots" + + return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" + + +def send_regression_notification( + analysis_result: dict, + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, + report_url: str = "", + tp: str = "", + dp: str = "", +) -> bool: + """Send regression/improvement Slack notification from analysis results. + + Uses a webhook URL (from *webhook_vault*) when available, falling + back to the Bot-Token path (from *notification_vault*). + + Args: + analysis_result: Output from run_regression_analysis() + model: Model name for display + accelerator: Accelerator name for display + job_id: Job identifier + slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + report_url: Optional presigned URL to an agent analysis report + tp: Tensor parallelism size for display + dp: Data parallelism size for display + + Returns: + True if notification sent successfully + """ + from projects.core.notifications.send import ( + _get_webhook_url, + _send_via_webhook, + send_notification, + ) + + status = analysis_result.get("status") + if status == "skipped": + reason = analysis_result.get("reason", "unknown") + logger.info("Regression analysis skipped: %s", reason) + return True + + if status != "completed": + return True + + regressions = analysis_result.get("regressions", []) + improvements = analysis_result.get("improvements", []) + + if not regressions and not improvements: + logger.info("No regressions or improvements to report") + return True + + current_version = analysis_result.get("current_version", "") + compare_version = analysis_result.get("compare_version", "") + + if regressions and improvements: + icon = ":warning:" + headline = "Performance regressions and improvements detected" + elif regressions: + icon = ":warning:" + headline = "Performance regressions detected" + else: + icon = ":large_green_circle:" + headline = "Performance improvements detected" + + detail_lines = [] + all_results = analysis_result.get("all_results", []) + profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + + for profile in profiles: + profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + detail_lines.append(f"\n*{profile}:*") + for r in profile_results: + if r.get("is_regression"): + direction = "dropped" if r["pct_diff"] < 0 else "increased" + detail_lines.append( + f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + else: + detail_lines.append( + f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + + details = "\n".join(detail_lines) + + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + + parallelism_line = "" + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + if parallelism_parts: + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" + + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + profiles=profiles if profiles else None, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + + message = ( + f"{icon} *{headline}*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"{report_line}" + f"{dashboard_line}" + f"*Changes:*\n{details}" + ) + + if dry_run: + logger.info("DRY RUN regression notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py index 69468b405..b1783b09c 100644 --- a/projects/rhaiis/toolbox/enable_profiler_gate/main.py +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -10,6 +10,17 @@ @entrypoint def run(*, name: str, namespace: str, gate_value: str = "0", disable: bool = False): + """Toggle the vLLM profiler gate file on a KServe predictor pod. + + Writes (or removes) /tmp/profiler_gate inside the predictor container + to enable or disable on-demand profiling for a running inference service. + + Args: + name: KServe InferenceService name used to locate the predictor pod. + namespace: Kubernetes namespace where the InferenceService is deployed. + gate_value: Value written to the gate file (default "0"). + disable: When True, removes the gate file instead of creating it. + """ return execute_tasks(locals()) diff --git a/vaults/psap-forge-dashboard-s3.yaml b/vaults/psap-forge-dashboard-s3.yaml index 145fc0e70..ada9243ed 100644 --- a/vaults/psap-forge-dashboard-s3.yaml +++ b/vaults/psap-forge-dashboard-s3.yaml @@ -1,15 +1,14 @@ env_key: PSAP_FORGE_DASHBOARD_S3_SECRET_PATH description: | - AWS credentials for rhaiis dashboard CSV sync (psap-dashboard-data) - and profiler trace uploads. Ported from model-furnace s3-uploader.conf. + AWS credentials for dashboard CSV sync and data uploads + to S3 buckets used by Forge projects. content: aws.credentials: description: | AWS credentials file (INI format). The key must have s3:GetObject, s3:PutObject, s3:CopyObject on: - - psap-dashboard-data (CSV dashboard, profiler traces) - - psap-model-furnace (vLLM logs) + - psap-dashboard-data (CSV dashboard data, profiler traces) sample: | [default] aws_access_key_id = AKIA... diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 12ae01cc8..5646df832 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -12,3 +12,8 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot + slack-webhook-url: + description: | + Plain-text file containing a Slack Incoming Webhook URL + for regression/comparison alerts. + Example: https://hooks.slack.com/services/T.../B.../... diff --git a/vaults/psap-forge-rhaiis-slack.yaml b/vaults/psap-forge-rhaiis-slack.yaml deleted file mode 100644 index 838d41964..000000000 --- a/vaults/psap-forge-rhaiis-slack.yaml +++ /dev/null @@ -1,10 +0,0 @@ -env_key: PSAP_FORGE_RHAIIS_SLACK_SECRET_PATH -description: | - Slack Incoming Webhook for rhaiis regression/comparison alerts. - Ported from model-furnace's slack.conf. - -content: - slack-webhook-url: - description: | - Plain-text file containing a single Slack Incoming Webhook URL. - Example: https://hooks.slack.com/services/T.../B.../... From a65612d3e4b1b9554844103e5506aadbc94c2bc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 14:09:49 +0200 Subject: [PATCH 0122/1072] [claude] Add hook for running ruff after file modifications --- .claude/settings.json | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..b9eecce75 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,36 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + }, + { + "matcher": "Edit", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + } + ] + } +} From d1752eb1db17ae1caa82564f0f840391fb513ed5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:10 +0200 Subject: [PATCH 0123/1072] [guidellm] toolbox: run_smoke_request: improve the troubleshooting when the Pod doesn't start --- .../toolbox/run_smoke_request/main.py | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index 703bef5fe..db3a7d742 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -5,7 +5,7 @@ import json import logging -from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, task from projects.core.dsl.utils import write_json, write_text from projects.core.dsl.utils.k8s import ( oc, @@ -129,7 +129,7 @@ def create_smoke_pod(args, ctx): return f"Created smoke pod {ctx.pod_name}" -@retry(attempts=60, delay=5, backoff=1.0) +@retry(attempts=12, delay=5, backoff=1.0) @task def wait_smoke_pod_running(args, ctx): """Wait for smoke pod to be running""" @@ -224,10 +224,61 @@ def execute_smoke_request(args, ctx): return "Smoke request completed successfully" +@always +@task +def capture_smoke_pod_debug_info(args, ctx): + """Capture pod YAML and description for debugging (always runs)""" + + if not hasattr(ctx, "pod_name"): + return "No pod name available for debug capture" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + try: + # Capture pod YAML + yaml_result = oc( + "get", + "pod", + ctx.pod_name, + "-n", + args.namespace, + "-o", + "yaml", + check=False, + ) + if yaml_result.returncode == 0 and yaml_result.stdout: + write_text(artifacts_dir / f"{ctx.pod_name}.yaml", yaml_result.stdout) + + # Capture pod description + desc_result = oc( + "describe", + "pod", + ctx.pod_name, + "-n", + args.namespace, + check=False, + ) + if desc_result.returncode == 0 and desc_result.stdout: + write_text(artifacts_dir / f"{ctx.pod_name}.description.txt", desc_result.stdout) + + return f"Captured debug info for pod {ctx.pod_name}" + + except Exception as e: + # Don't fail the task if debug capture fails + logger.warning(f"Failed to capture debug info for pod {ctx.pod_name}: {e}") + return f"Failed to capture debug info: {e}" + + +@always @task def cleanup_smoke_pod(args, ctx): """Delete the smoke pod after test completion""" + if not hasattr(ctx, "pod_name"): + return "No pod name available for cleanup" + oc( "delete", "pod", From 0117b588ffd991a1f0e80c1fa1c6ccab5a09a936 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:30 +0200 Subject: [PATCH 0124/1072] [guidellm] toolbox: run_smoke_request: use a FQDN quay image for curl --- projects/guidellm/toolbox/run_smoke_request/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index db3a7d742..fda906902 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -24,7 +24,7 @@ def run( namespace: str, endpoint_url: str, pod_name: str = "llm-d-smoke", - client_image: str = "curlimages/curl:8.11.1", + client_image: str = "quay.io/curl/curl:8.21.0", endpoint_path: str = "/v1/completions", request_timeout_seconds: int = 60, served_model_name: str, From c16641230063daf725084b63f0a549c8e4fcf888 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:19:29 +0200 Subject: [PATCH 0125/1072] [rhoai] library: deploy: add the _rhoai suffix when deploying the custom catalog --- projects/rhoai/library/deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index 91c147547..f62bd84ba 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -125,6 +125,7 @@ def deploy_rhoai_custom_catalog(*, custom_catalog: RhoaiCustomCatalogConfig) -> catalog_image=custom_catalog.image, display_name=custom_catalog.display_name or custom_catalog.name, publisher=custom_catalog.publisher or "", + artifact_dirname_suffix="rhoai", ) From df4e2ab4b961e68260d84103789b820bc85c55b8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:18:53 +0200 Subject: [PATCH 0126/1072] [llm_d] orchestration: config: use a FQDN quay image for curl --- projects/llm_d/orchestration/config.d/platform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index b9defea06..32325769a 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -92,6 +92,6 @@ artifacts: smoke: pod_name: llm-d-smoke - client_image: curlimages/curl:8.11.1 + client_image: quay.io/curl/curl:8.21.0 endpoint_path: /v1/completions request_timeout_seconds: 60 From 3fde22169b74082f63a2374a7e59b04d6a8762e4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:38:36 +0200 Subject: [PATCH 0127/1072] [core] library: config: add a helper method to generate the variables_override file --- projects/core/library/config.py | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index d06fca5e2..d0d0437cb 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -519,3 +519,57 @@ def reload(orchestration_dir, *, apply_config_overrides=True): init(orchestration_dir, apply_config_overrides=apply_config_overrides) return project + + +def write_variables_override(presets=None, variables_dict=None): + """Write configuration overrides to the variables_override.yaml file in ARTIFACT_DIR. + + This is a top-level function that doesn't require project config to be initialized. + + Args: + presets: List of presets to set in project.args + variables_dict: Dictionary of additional configuration paths and values + """ + from pathlib import Path + + if not env.ARTIFACT_DIR: + raise ValueError( + "env.ARTIFACT_DIR is not initialized. " + "Call env.init() before using write_variables_override()" + ) + + artifact_dir = Path(env.ARTIFACT_DIR) + override_file = artifact_dir / VARIABLE_OVERRIDES_FILENAME + + # Ensure the directory exists + override_file.parent.mkdir(parents=True, exist_ok=True) + + # Load existing overrides or create empty structure + if override_file.exists(): + with open(override_file) as f: + loaded = yaml.safe_load(f) + if loaded is not None and not isinstance(loaded, dict): + raise ValueError( + f"Override file {override_file} must contain a YAML dictionary, got {type(loaded).__name__}" + ) + overrides = loaded or {} + else: + overrides = {} + + # Set presets in project.args + if presets: + overrides["project.args"] = presets + logger.info(f"write_variables_override: project.args --> {presets}") + + # Add additional variables to top level + if variables_dict: + for jsonpath, value in variables_dict.items(): + overrides[jsonpath] = value + logger.info(f"write_variables_override: {jsonpath} --> {value}") + + # Write back to file + with open(override_file, "w") as f: + yaml.dump(overrides, f, indent=4, default_flow_style=False, sort_keys=False) + + total_items = (1 if presets else 0) + (len(variables_dict) if variables_dict else 0) + logger.info(f"write_variables_override: wrote {total_items} override(s) to {override_file}") From 9142731938d5b5a46a8b9b156e13e82fa78c5c01 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:39:37 +0200 Subject: [PATCH 0128/1072] [llm_d] orchestration: ci: allow passing the presets to apply before launching a command --- projects/llm_d/CHANGELOG-orchestration.md | 16 ++++++++++++++++ projects/llm_d/orchestration/ci.py | 20 +++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/CHANGELOG-orchestration.md b/projects/llm_d/CHANGELOG-orchestration.md index 5ccfc73a7..46fdeddcd 100644 --- a/projects/llm_d/CHANGELOG-orchestration.md +++ b/projects/llm_d/CHANGELOG-orchestration.md @@ -1,5 +1,21 @@ # LLM-D Orchestration Changelog +## 2026-07-16 - Preset Configuration Support + +### Configuration Management +- **Preset CLI Option**: Added `--preset` flag to CI entrypoint for preset configuration before execution + - **Multiple Presets**: Supports multiple preset values via repeated `--preset` flags + - **Early Initialization**: Sets preset configuration during environment initialization, before config loading + - **Variables Override Integration**: Uses framework helper to generate proper override files + +### Files Modified +- `projects/llm_d/orchestration/ci.py` - Added `--presets` CLI option and initialization logic + +### Benefits +- **Flexible Testing**: Enables running same test suite with different preset configurations +- **Command-Line Control**: Provides runtime preset selection without modifying configuration files +- **Framework Integration**: Leverages core configuration override system for consistent behavior + ## 2026-06-26 - Orchestration & Cleanup Improvements ### New Features diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index b3748f679..25398162a 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -34,10 +34,15 @@ ] -def init(): +def init(presets=None): """Initialize LLM-D orchestration environment""" env.init() run.init() + + # Set presets configuration if provided + if presets: + config.write_variables_override(presets=presets) + config.init(Path(__file__).parent) @@ -69,12 +74,21 @@ def init_vaults_for_phase(phase: str) -> None: @click.group(cls=ci_lib.HelpfulGroup) +@click.option("--preset", multiple=True, help="Set preset configuration before starting") @click.pass_context @ci_lib.safe_ci_function -def main(ctx): +def main(ctx, preset): """LLM-D Project CI Operations for FORGE.""" ctx.ensure_object(types.SimpleNamespace) - init() + + presets_list = list(preset) + if presets_list and "," in presets_list[0]: + lst = presets_list.pop(0) + presets_list = lst.split(",") + presets_list + + logger.info(f"Setting preset configuration from CLI: {presets_list}") + + init(presets_list) if ctx.invoked_subcommand == "resolve-fournos-config": logger.info("No need to initialize the vaults for the resolve step") From e044d84dda329dd4affb0fb421277660404d0a80 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Fri, 17 Jul 2026 14:36:15 +0100 Subject: [PATCH 0129/1072] feat: Do not retry Signed-off-by: Alberto Perdomo --- .../toolbox/cleanup_test_resources/main.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 9afa1fb88..475bf49ec 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -6,7 +6,7 @@ import subprocess from projects.core.dsl import entrypoint, execute_tasks, retry, shell, task -from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_get_json logger = logging.getLogger("DSL") @@ -164,35 +164,38 @@ def delete_llm_d_labeled_resources(args, ctx): def delete_inference_service(args, ctx): """Delete the LLM inference service""" - _best_effort_delete( - "llminferenceservice", + result = oc( "delete", "llminferenceservice", args.inference_service_name, "-n", args.namespace, "--ignore-not-found=true", + "--timeout=30s", + check=False, ) + if result.returncode != 0: + logger.error( + "Failed to delete llminferenceservice/%s (rc=%d), capturing describe", + args.inference_service_name, + result.returncode, + ) + oc( + "describe", + "llminferenceservice", + args.inference_service_name, + "-n", + args.namespace, + check=False, + ) + raise RuntimeError( + f"Failed to delete llminferenceservice/{args.inference_service_name} " + f"in {args.namespace} (rc={result.returncode})" + ) return f"Deleted llminferenceservice {args.inference_service_name}" -@retry(attempts=90, delay=10, backoff=1.0) -@task -def wait_for_inference_service_deletion(args, ctx): - """Wait for the llminferenceservice to be deleted""" - - if not oc_resource_exists( - "llminferenceservice", args.inference_service_name, namespace=args.namespace - ): - return f"llminferenceservice/{args.inference_service_name} deleted from {args.namespace}" - - return ( - False, - f"Waiting for llminferenceservice/{args.inference_service_name} deletion in {args.namespace}", - ) - - @retry(attempts=90, delay=10, backoff=1.0) @task def wait_for_workload_pods_deletion(args, ctx): From cbc8a59db076dbd2ad32531755b18620805dee25 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 17 Jul 2026 13:13:08 -0400 Subject: [PATCH 0130/1072] Make HF_TOKEN secret name configurable in guidellm benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The secret name was hardcoded to "storage-config". Now it's driven by the benchmarks.guidellm.hf_token_secret config key and conditionally injected—if the key is empty or omitted, no HF_TOKEN env var is added to the K8s job manifest. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 3 +++ .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 4 +++- projects/guidellm/toolbox/run_guidellm_benchmark/utils.py | 3 +++ projects/rhaiis/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 3 +++ 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 5a8253abf..1d301988b 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -33,6 +33,7 @@ def run( timeout: int = 900, pvc_size: str = "1Gi", guidellm_args: list[str] | None = None, + hf_token_secret: str = "", ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -45,6 +46,7 @@ def run( timeout: Timeout in seconds to wait for job completion pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. """ execute_tasks(locals()) @@ -139,6 +141,7 @@ def create_guidellm_resources_task(args, ctx): image=ctx.image, endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, + hf_token_secret=args.hf_token_secret, ), ) return f"GuideLLM benchmark {ctx.benchmark_name} created" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 7a74cdd08..8f1ef15b6 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -26,12 +26,14 @@ spec: env: - name: USER value: guidellm +{% if hf_token_secret %} - name: HF_TOKEN valueFrom: secretKeyRef: - name: storage-config + name: {{ hf_token_secret }} key: HF_TOKEN optional: true +{% endif %} volumeMounts: - name: home mountPath: /home/guidellm diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 55e81d035..7cab6a2c2 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -175,6 +175,7 @@ def render_guidellm_job_from_parts( image: str, endpoint_url: str, guidellm_args: list[str], + hf_token_secret: str = "", ) -> dict[str, Any]: """Render a GuideLL-M job manifest from individual components. @@ -184,6 +185,7 @@ def render_guidellm_job_from_parts( image: Container image for GuideLLM endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. Returns: Job manifest as dict @@ -195,6 +197,7 @@ def render_guidellm_job_from_parts( "namespace": namespace, "name": name, "image": image, + "hf_token_secret": hf_token_secret, }, ) manifest = yaml.safe_load(rendered_yaml) diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 1442b8c3a..c0477ff2f 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -11,6 +11,7 @@ benchmarks: image: ghcr.io/vllm-project/guidellm:v0.6.0 timeout: 14400 pvc_size: 10Gi + hf_token_secret: "storage-config" args: backend_type: openai_http rate_type: concurrent diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 20038e4bd..180cd46e7 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -307,6 +307,7 @@ def _run_workload_benchmark( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) from pathlib import Path candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) @@ -824,6 +825,7 @@ def _run_warmup_step( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) logger.info("Warmup completed") except Exception: @@ -888,6 +890,7 @@ def _run_profiler_step( timeout=benchmark_timeout, pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) finally: enable_profiler_gate( From bdba8e98895583e50b822ab7f5ab6433139f9c80 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 14:17:21 +0200 Subject: [PATCH 0131/1072] [kserve] toolbox: deploy_llmisvc: accept a dry-run parameter to only build the llmisvc src --- .../kserve/toolbox/deploy_llmisvc/main.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 5b27231bf..a34c14412 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -6,7 +6,15 @@ import yaml -from projects.core.dsl import always, entrypoint, execute_tasks, on_failure, retry, task +from projects.core.dsl import ( + EarlyReturn, + always, + entrypoint, + execute_tasks, + on_failure, + retry, + task, +) from projects.core.dsl.utils import write_text from projects.core.dsl.utils.k8s import ( oc, @@ -28,6 +36,7 @@ def run( namespace: str, inference_service_manifest_path: str, gateway_status_address_name: str = "gateway-external", + dry_run: bool = False, ) -> str: """ Deploy an LLMInferenceService and wait for its endpoint. @@ -36,6 +45,7 @@ def run( namespace: Namespace used by llm_d inference_service_manifest_path: Path to the InferenceService YAML manifest file gateway_status_address_name: Gateway status address name for endpoint resolution + dry_run: If True, only prepare the manifest without deploying """ # Load manifest to extract the service name @@ -48,9 +58,13 @@ def run( "inference_service_manifest_path": inference_service_manifest_path, "inference_service_name": inference_service_name, "gateway_status_address_name": gateway_status_address_name, + "dry_run": dry_run, } context = execute_tasks(task_args) + if dry_run: + return "" + # Ensure endpoint_url is available endpoint_url = getattr(context, "endpoint_url", None) if not endpoint_url: @@ -86,6 +100,16 @@ def copy_manifest_to_src(args, ctx): return f"Copied manifest from {original_path} to {src_path}" +@task +def check_dry_run(args, ctx): + """Check if dry-run mode is enabled and return early if so""" + if args.dry_run: + return EarlyReturn( + f"Dry-run completed: Prepared manifest for {args.inference_service_name}" + ) + return "Proceeding with full deployment" + + @task def delete_existing_service(args, ctx): """Delete existing LLMInferenceService""" @@ -154,6 +178,9 @@ def wait_pods_appear(args, ctx): def capture_llmisv_description(args, ctx): """Capture LLMISV description with events and status for failure analysis""" + if args.dry_run: + return "Dry-run, nothing to do" + try: # Ensure artifacts directory exists artifacts_dir = args.artifact_dir / "artifacts" @@ -188,6 +215,9 @@ def capture_llmisv_description(args, ctx): def capture_replicaset_description(args, ctx): """Capture ReplicaSet description for pod creation failure analysis""" + if args.dry_run: + return "Dry-run, nothing to do" + try: # Ensure artifacts directory exists artifacts_dir = args.artifact_dir / "artifacts" From 8e7bfaa2253706913c5272e769b115d35c0d3db5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:11:00 +0200 Subject: [PATCH 0132/1072] [kserve] toolbox: deploy_llmisvc: update to avoid building a custom dict for the args --- .../kserve/toolbox/deploy_llmisvc/main.py | 124 +++++++----------- 1 file changed, 48 insertions(+), 76 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index a34c14412..cf064757c 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -48,44 +48,32 @@ def run( dry_run: If True, only prepare the manifest without deploying """ - # Load manifest to extract the service name - manifest = load_yaml(Path(inference_service_manifest_path)) - inference_service_name = manifest["metadata"]["name"] - - # Pass only the required arguments to avoid including manifest content in logs - task_args = { - "namespace": namespace, - "inference_service_manifest_path": inference_service_manifest_path, - "inference_service_name": inference_service_name, - "gateway_status_address_name": gateway_status_address_name, - "dry_run": dry_run, - } - context = execute_tasks(task_args) + ctx = execute_tasks(locals()) if dry_run: - return "" + return ctx.src_manifest_path # Ensure endpoint_url is available - endpoint_url = getattr(context, "endpoint_url", None) + endpoint_url = getattr(ctx, "endpoint_url", None) if not endpoint_url: raise RuntimeError("Failed to resolve gateway endpoint URL after deployment") - # Log service description for visibility - service_description = getattr(context, "service_description", None) - if service_description: - print(f"Deployed: {service_description}") - return endpoint_url @task def copy_manifest_to_src(args, ctx): - """Copy inference service manifest to src directory for inspection and use""" + """Copy inference service manifest to src directory and extract service name""" import shutil # Get the original manifest path original_path = Path(args.inference_service_manifest_path) + # Load manifest to extract the service name + manifest = load_yaml(original_path) + ctx.inference_service_name = manifest["metadata"]["name"] + ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}" + # Ensure the src directory exists src_dir = args.artifact_dir / "src" src_dir.mkdir(parents=True, exist_ok=True) @@ -97,16 +85,14 @@ def copy_manifest_to_src(args, ctx): # Store the src path in context for other tasks to use ctx.src_manifest_path = str(src_path) - return f"Copied manifest from {original_path} to {src_path}" + return f"Copied manifest from {original_path} to {src_path} (service: {ctx.inference_service_name})" @task def check_dry_run(args, ctx): """Check if dry-run mode is enabled and return early if so""" if args.dry_run: - return EarlyReturn( - f"Dry-run completed: Prepared manifest for {args.inference_service_name}" - ) + return EarlyReturn(f"Dry-run completed: Prepared manifest for {ctx.inference_service_name}") return "Proceeding with full deployment" @@ -114,7 +100,7 @@ def check_dry_run(args, ctx): def delete_existing_service(args, ctx): """Delete existing LLMInferenceService""" - name = args.inference_service_name + name = ctx.inference_service_name oc( "delete", "llminferenceservice", @@ -124,8 +110,7 @@ def delete_existing_service(args, ctx): "--ignore-not-found=true", check=False, ) - ctx.service_name = name - ctx.selector = f"app.kubernetes.io/name={name}" + return f"Deleted existing LLMInferenceService {name}" @@ -134,13 +119,11 @@ def delete_existing_service(args, ctx): def wait_old_pods_gone(args, ctx): """Wait for old llm-d pods to disappear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") - - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) + pods = oc_get_json( + "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + ) if not pods or not pods.get("items"): - return f"Old pods gone for {service_name}" + return f"Old pods gone for {ctx.inference_service_name}" return False # Retry @@ -154,7 +137,7 @@ def apply_inference_service(args, ctx): # Load and apply the manifest from src manifest = load_yaml(Path(src_manifest_path)) oc_apply(src_manifest_path, manifest) - return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.service_name}" + return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.inference_service_name}" @on_failure(on_wait_pods_appear_failure) @@ -163,13 +146,11 @@ def apply_inference_service(args, ctx): def wait_pods_appear(args, ctx): """Wait for llm-d pods to appear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") - - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) + pods = oc_get_json( + "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + ) if pods and pods.get("items"): - return f"Pods appeared for {service_name}" + return f"Pods appeared for {ctx.inference_service_name}" return False # Retry @@ -186,8 +167,10 @@ def capture_llmisv_description(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" # Capture LLMISV description result = oc( @@ -223,16 +206,17 @@ def capture_replicaset_description(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" # Get replicasets for the service rs_result = oc( "get", "replicaset", "-l", - selector, + ctx.selector, "-n", args.namespace, "-o", @@ -246,16 +230,18 @@ def capture_replicaset_description(args, ctx): if rs_result.stdout.strip(): # Describe each replicaset for rs_name in rs_result.stdout.strip().split("\n"): - if rs_name.strip(): - rs_desc_result = oc( - "describe", - rs_name.strip(), - "-n", - args.namespace, - log_stdout=False, - check=False, - ) - replicaset_descriptions.append(rs_desc_result.stdout) + if not rs_name.strip(): + continue + + rs_desc_result = oc( + "describe", + rs_name.strip(), + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + replicaset_descriptions.append(rs_desc_result.stdout) # Save all replicaset descriptions rs_desc_path = artifacts_dir / "replicaset_description.txt" @@ -275,7 +261,7 @@ def capture_replicaset_description(args, ctx): def query_service_status(args, ctx): """Query the status of the LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query only the Ready condition status result = oc( @@ -302,7 +288,7 @@ def query_service_status(args, ctx): def query_service_message(args, ctx): """Query detailed message from LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query the Ready condition details result = oc( @@ -339,8 +325,7 @@ def query_service_message(args, ctx): def wait_service_ready(args, ctx): """Wait for LLMInferenceService to be ready""" - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + service_name = ctx.inference_service_name # Query the current status and show diagnostic info result = oc( @@ -359,7 +344,7 @@ def wait_service_ready(args, ctx): "get", "pods", "-l", - selector, + ctx.selector, "-n", args.namespace, log_stdout=True, # Show pod status in logs @@ -395,7 +380,7 @@ def resolve_endpoint_task(args, ctx): endpoint_url = try_resolve_endpoint_url( namespace=args.namespace, - inference_service_name=args.inference_service_name, + inference_service_name=ctx.inference_service_name, gateway_status_address_name=args.gateway_status_address_name, ) if endpoint_url: @@ -405,19 +390,6 @@ def resolve_endpoint_task(args, ctx): return False # Retry -@task -def deploy_llmisvc_task(args, ctx): - """Deploy the llm_d inference service and resolve its endpoint""" - - # All work is done by the individual tasks - service_info = getattr( - ctx, "service_description", f"LLM Service '{args.inference_service_name}'" - ) - return ( - f"LLMInferenceService deployment completed: {service_info} - Endpoint: {ctx.endpoint_url}" - ) - - def try_resolve_endpoint_url( *, namespace: str, inference_service_name: str, gateway_status_address_name: str ) -> str | None: From 76de0e39d1015c065f1885ea903dd0f8bd4b7539 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 14:27:02 +0200 Subject: [PATCH 0133/1072] [llm_d] orchestration: add a kserve dry-run flag --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 80 ++++++++++++------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 6dd9496ca..95ac92ebb 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,3 +5,4 @@ smoke_request_key: default benchmark_key: null job_name: null namespace_override: null +kserve_dry_run: false diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2ef047ef2..451043985 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -7,7 +7,7 @@ from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier -from projects.core.library import env +from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt from projects.core.orchestration.utils.k8s import ensure_namespace @@ -17,6 +17,7 @@ from projects.kserve.toolbox.capture_llmisvc_state import main as capture_llmisvc_state from projects.kserve.toolbox.deploy_llmisvc import main as deploy_llmisvc from projects.kserve.toolbox.wait_kserve_ready import main as wait_kserve_ready +from projects.llm_d.orchestration import runtime_config from projects.llm_d.orchestration.prepare_phase import prepare_model_cache from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, @@ -29,7 +30,6 @@ def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() deployment_profile = runtime_config.get_deployment_profile_name() @@ -52,6 +52,13 @@ def create_test_labels() -> None: def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" + + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + if dry_run: + ret = do_test() + logger.info("Kserve dry-run mode enabled - Skipping caliper post-processing") + return ret + return run_and_postprocess(do_test) @@ -74,8 +81,6 @@ def _run_finalizer( logger.exception("Ignoring %s failure after primary test failure", description) return finalizer_exc - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() capture_namespace_events = platform["artifacts"]["capture_namespace_events"] @@ -107,18 +112,19 @@ def _run_finalizer( def do_test() -> int: # Load minimal config needed for orchestration flow - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() - - # Ensure namespace exists before starting any deployments - ensure_namespace( - namespace, - labels={ - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - }, - ) + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + + if not dry_run: + # Ensure namespace exists before starting any deployments + ensure_namespace( + namespace, + labels={ + "app.kubernetes.io/managed-by": "forge", + "forge.openshift.io/project": "llm_d", + }, + ) endpoint_url: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None @@ -131,6 +137,10 @@ def do_test() -> int: endpoint_url = deploy_inference_service() + if dry_run: + logging.warning("Running in dry-run mode, skipping the rest of the test steps") + return 0 + if not endpoint_url: raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") run_smoke_request(endpoint_url=endpoint_url) @@ -146,6 +156,9 @@ def do_test() -> int: logging.warning("Caught a SignalInterrupt, skipping the finalizers") do_finalizers = False + if dry_run: + do_finalizers = False + if do_finalizers: primary_exc, finalizer_exc = run_finalizers( endpoint_url, primary_exc, finalizer_exc @@ -169,37 +182,50 @@ def deploy_inference_service() -> str: logger.info("Starting LLMInferenceService deployment") # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # Step 1: Ensure model cache is ready - _prepare_model_cache() + # Check if dry-run mode is enabled early + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + + # Step 1: Ensure model cache is ready (skip in dry-run) + if not dry_run: + _prepare_model_cache() + else: + logger.info("Skipping model cache preparation - dry-run mode enabled") # Step 2: Wait for the serving control plane to settle before creating the service. - rhoai_namespace = platform["rhoai"]["namespace"] - wait_kserve_ready.run(namespace=rhoai_namespace) + if not dry_run: + rhoai_namespace = platform["rhoai"]["namespace"] + wait_kserve_ready.run(namespace=rhoai_namespace) # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) + endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), gateway_status_address_name=gateway["status_address_name"], + dry_run=dry_run, ) + if dry_run: + llmisvc_manifest_path = endpoint_url + logger.info("Dry-run completed: LLMInferenceService manifest prepared:") + logger.info(llmisvc_manifest_path) + return llmisvc_manifest_path + logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url def _prepare_model_cache() -> None: """Ensure model cache PVC is ready for deployment.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() logger.info("Preparing model cache for model: %s", model_name) @@ -211,7 +237,6 @@ def _prepare_model_cache() -> None: def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" - from projects.llm_d.orchestration import runtime_config config_dir = runtime_config.get_config_dir() namespace = runtime_config.get_namespace() @@ -245,7 +270,6 @@ def _build_inference_service_manifest() -> Path: def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() @@ -267,9 +291,6 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: def run_guidellm_benchmark(*, endpoint_url: str) -> None: - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() benchmark_configs = runtime_config.get_benchmark_configs() @@ -295,7 +316,6 @@ def run_guidellm_benchmark(*, endpoint_url: str) -> None: def capture_inference_service_state() -> None: # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() @@ -318,8 +338,12 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config + + # Skip cleanup when in dry-run mode + dry_run = config.project.get_config("runtime.kserve_dry_run", False) + if dry_run: + logger.info("Skipping cleanup_test_resources - dry-run mode enabled") + return namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() From 9b6ab212990beefabf867803461b9385683c5392 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:17:26 +0200 Subject: [PATCH 0134/1072] [guidellm] toolbox: run_smoke_request: save to file directly --- projects/guidellm/toolbox/run_smoke_request/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index fda906902..5905beff2 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -238,7 +238,7 @@ def capture_smoke_pod_debug_info(args, ctx): try: # Capture pod YAML - yaml_result = oc( + oc( "get", "pod", ctx.pod_name, @@ -248,20 +248,17 @@ def capture_smoke_pod_debug_info(args, ctx): "yaml", check=False, ) - if yaml_result.returncode == 0 and yaml_result.stdout: - write_text(artifacts_dir / f"{ctx.pod_name}.yaml", yaml_result.stdout) # Capture pod description - desc_result = oc( + oc( "describe", "pod", ctx.pod_name, "-n", args.namespace, check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.description.txt", ) - if desc_result.returncode == 0 and desc_result.stdout: - write_text(artifacts_dir / f"{ctx.pod_name}.description.txt", desc_result.stdout) return f"Captured debug info for pod {ctx.pod_name}" From c02d8e34f56030b547b33e537b34a4aaa1745dfe Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 13:00:58 +0300 Subject: [PATCH 0135/1072] feat: add nightly pipeline to forge using clusterless pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a clusterless nightly pipeline that automatically detects new image versions and triggers test runs when a new version is found. Architecture: - Core nightly package (projects/core/nightly/) defines abstract interfaces (ImageReceiver, NightlyVerifier) with built-in retry logic. Projects implement these to define how versions are fetched and how run history is checked. - Phase handlers (receive_image.py, confirm.py) use polymorphism to call the project's implementations without knowing the concrete types. - Auto-discovery in the project's resolvers/ package — adding a new source is just dropping a new file with a NAME attribute. mcp_gateway implementation: - GHCRReceiver: finds the latest commit with a published ghcr.io image - MLflowVerifier: queries MLflow experiment runs to find the last tested version - CronJob triggers a clusterless FournosJob nightly Pipeline flow: 1. CronJob creates a clusterless FournosJob with pipeline: "nightly" 2. receive-image task: calls project's get_receiver().get_latest_version() 3. confirm-nightly task: calls project's get_verifier().get_last_tested_version(), compares versions, creates a full FournosJob if new version detected Also generalizes task-forge-step env var export to loop over all spec.env keys instead of hardcoding specific variable names. Co-authored-by: Cursor --- .../gitops/base/workflows/kustomization.yaml | 2 + .../base/workflows/pipeline-nightly.yaml | 63 +++++++++ .../base/workflows/task-forge-step.yaml | 8 +- projects/agentic_tools/ci_base.py | 17 ++- projects/core/nightly/__init__.py | 22 +++ projects/core/nightly/base_receiver.py | 46 ++++++ projects/core/nightly/base_verifier.py | 46 ++++++ projects/core/nightly/confirm.py | 131 ++++++++++++++++++ projects/core/nightly/receive_image.py | 42 ++++++ .../mcp_gateway/orchestration/config.yaml | 7 + projects/mcp_gateway/resolvers/__init__.py | 66 +++++++++ .../mcp_gateway/resolvers/cronjob-ghcr.yaml | 50 +++++++ .../mcp_gateway/resolvers/ghcr_receiver.py | 70 ++++++++++ .../mcp_gateway/resolvers/mlflow_verifier.py | 121 ++++++++++++++++ 14 files changed, 686 insertions(+), 5 deletions(-) create mode 100644 fournos/gitops/base/workflows/pipeline-nightly.yaml create mode 100644 projects/core/nightly/__init__.py create mode 100644 projects/core/nightly/base_receiver.py create mode 100644 projects/core/nightly/base_verifier.py create mode 100644 projects/core/nightly/confirm.py create mode 100644 projects/core/nightly/receive_image.py create mode 100644 projects/mcp_gateway/resolvers/__init__.py create mode 100644 projects/mcp_gateway/resolvers/cronjob-ghcr.yaml create mode 100644 projects/mcp_gateway/resolvers/ghcr_receiver.py create mode 100644 projects/mcp_gateway/resolvers/mlflow_verifier.py diff --git a/fournos/gitops/base/workflows/kustomization.yaml b/fournos/gitops/base/workflows/kustomization.yaml index 2212ab4a0..2bdff8912 100644 --- a/fournos/gitops/base/workflows/kustomization.yaml +++ b/fournos/gitops/base/workflows/kustomization.yaml @@ -6,10 +6,12 @@ resources: - task-forge-step.yaml - pipeline-full.yaml - pipeline-launcher.yaml + - pipeline-nightly.yaml - pipeline-prepare-only.yaml - pipeline-prepare-test.yaml - pipeline-replot.yaml - pipeline-test-only.yaml + commonLabels: app.kubernetes.io/name: forge-on-fournos app.kubernetes.io/component: workflows diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml new file mode 100644 index 000000000..18f729de1 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -0,0 +1,63 @@ +# Nightly pipeline: clusterless job that checks for new image versions +# and conditionally triggers a full test FournosJob. +# +# Flow: receive-image (resolve latest version) → confirm-nightly (compare with S3, create FournosJob if new) +# +# This pipeline runs as a clusterless FournosJob (no target cluster needed). +# Both tasks use the standard forge-step, dispatching to phases registered +# in the project's ci.py (backed by shared handlers in projects/core/nightly/). +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: nightly + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-receive-image + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: receive-image + - name: job-step-name + value: 00__receive-image + + - name: 01-confirm-nightly + runAfter: [00-receive-image] + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: confirm-nightly + - name: job-step-name + value: 01__confirm-nightly diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index 90f746be6..d8d9e0fc9 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -106,10 +106,10 @@ spec: fi } - # Export environment variables from FournosJob spec - export_env_var_from_fjob PULL_BASE_SHA - export_env_var_from_fjob PULL_PULL_SHA - export_env_var_from_fjob PULL_NUMBER + # Export all environment variables from FournosJob spec.env + for var_name in $(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r '.spec.env // {} | keys[]'); do + export_env_var_from_fjob "$var_name" + done if [[ "${PULL_PULL_SHA:-}" ]]; then echo diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index d868f6fc6..853848660 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -134,7 +134,7 @@ def list_vaults(self) -> list[str]: def should_init_vaults(self, phase: str) -> bool: """Return False to skip vault init for certain phases.""" - return phase != "resolve-fournos-config" + return phase not in ("resolve-fournos-config", "receive-image", "confirm-nightly") def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. @@ -176,6 +176,17 @@ class path like ``projects.foo.notifications.MyProvider``) and # Build # ------------------------------------------------------------------ + _NIGHTLY_PHASES: dict[str, PhaseSpec] = { + "receive-image": PhaseSpec( + func="projects.core.nightly.receive_image:run", + help="Nightly: resolve latest available image version.", + ), + "confirm-nightly": PhaseSpec( + func="projects.core.nightly.confirm:run", + help="Nightly: check S3 for last run, create FournosJob if new version.", + ), + } + def build(self) -> click.Group: """Assemble and return the fully-wired click.Group.""" app = self @@ -200,6 +211,10 @@ def main(ctx): for cmd_name, spec in self.phases.items(): _register_phase_command(main, cmd_name, spec) + for cmd_name, spec in self._NIGHTLY_PHASES.items(): + if cmd_name not in self.phases: + _register_phase_command(main, cmd_name, spec) + main.add_command( create_fournos_resolve_entrypoint( vault_list_func=self.list_vaults, diff --git a/projects/core/nightly/__init__.py b/projects/core/nightly/__init__.py new file mode 100644 index 000000000..183c607a2 --- /dev/null +++ b/projects/core/nightly/__init__.py @@ -0,0 +1,22 @@ +"""Shared nightly pipeline handlers. + +Architecture: + Core (this package) defines abstract interfaces and the orchestration logic. + Each project implements concrete receivers/verifiers in its resolvers/ package + and exposes them via get_receiver() and get_verifier() factory functions. + + The core handlers import the project's resolvers module at runtime and call + those factories — pure polymorphism, no knowledge of concrete implementations. + +Interfaces: + - ImageReceiver: fetches the latest version from a registry + - NightlyVerifier: checks if a version was already tested + +Handlers: + - receive_image.run(): calls project's get_receiver().get_latest_version() + - confirm.run(): calls project's get_verifier().get_last_tested_version() + +Projects register these as phases in ci_base: + "receive-image" -> projects.core.nightly.receive_image:run + "confirm-nightly" -> projects.core.nightly.confirm:run +""" diff --git a/projects/core/nightly/base_receiver.py b/projects/core/nightly/base_receiver.py new file mode 100644 index 000000000..ba93c82c0 --- /dev/null +++ b/projects/core/nightly/base_receiver.py @@ -0,0 +1,46 @@ +"""Abstract interface for image receivers. + +Projects implement this to define how the latest available version +is fetched from a container registry (GHCR, Quay, etc.). +""" + +from __future__ import annotations + +import logging +import time +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + + +class ImageReceiver(ABC): + """Fetches the latest available version from a container registry.""" + + @abstractmethod + def _fetch_version(self) -> str: + """Query the registry and return the latest version string. + + Raises on failure (network, auth, no tags found, etc.). + """ + ... + + def get_latest_version(self) -> str: + """Call _fetch_version with retry on transient failures.""" + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_version() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "Receiver attempt %d/%d failed: %s. Retrying in %ds...", + attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError( + f"Receiver failed after {MAX_ATTEMPTS} attempts" + ) from last_error diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py new file mode 100644 index 000000000..8d1212aa1 --- /dev/null +++ b/projects/core/nightly/base_verifier.py @@ -0,0 +1,46 @@ +"""Abstract interface for nightly verifiers. + +Projects implement this to define how run history is checked +(S3, MLflow, etc.) to determine if a version was already tested. +""" + +from __future__ import annotations + +import logging +import time +from abc import ABC, abstractmethod + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + + +class NightlyVerifier(ABC): + """Checks historical run data to determine if a version was already tested.""" + + @abstractmethod + def _fetch_last_version(self) -> str: + """Return the version string of the last successful run. + + Returns empty string if no previous run exists. + """ + ... + + def get_last_tested_version(self) -> str: + """Call _fetch_last_version with retry on transient failures.""" + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_last_version() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "Verifier attempt %d/%d failed: %s. Retrying in %ds...", + attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError( + f"Verifier failed after {MAX_ATTEMPTS} attempts" + ) from last_error diff --git a/projects/core/nightly/confirm.py b/projects/core/nightly/confirm.py new file mode 100644 index 000000000..b677b0176 --- /dev/null +++ b/projects/core/nightly/confirm.py @@ -0,0 +1,131 @@ +"""Nightly confirm handler. + +Loads the project's resolvers module to get a NightlyVerifier instance. +Compares the resolved version (from receive-image step) against what +the verifier reports as the last tested version. Creates a FournosJob +if a new version is detected. + +The core never knows how or where history is stored — it just calls +get_last_tested_version() on whatever the project provides. +""" + +from __future__ import annotations + +import importlib +import json +import logging +import os +import subprocess +from pathlib import Path + +from projects.core.library import config + +logger = logging.getLogger(__name__) + + +def _read_resolved_version() -> str: + """Read the version written by the receive-image step.""" + artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) + version_file = artifact_base / "resolved_version.txt" + + if not version_file.exists(): + raise FileNotFoundError( + f"resolved_version.txt not found at {version_file}. " + "Did the receive-image step run successfully?" + ) + + version = version_file.read_text().strip() + if not version: + raise RuntimeError("resolved_version.txt is empty") + return version + + +def _create_fournos_job(project: str, version: str) -> None: + """Create a FournosJob for the full test run.""" + namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") + cluster = config.project.get_config("nightly.cluster") + version_key = config.project.get_config("nightly.version_key") + pipeline = config.project.get_config("nightly.pipeline") + preset = config.project.get_config("nightly.preset", "") + owner = config.project.get_config("nightly.owner", "nightly-pipeline") + + args_list = [preset] if preset else [] + + fjob_manifest = { + "apiVersion": "fournos.dev/v1", + "kind": "FournosJob", + "metadata": { + "generateName": f"nightly-{project.replace('_', '-')}-", + "namespace": namespace, + }, + "spec": { + "cluster": cluster, + "displayName": f"nightly {project} {version}", + "owner": owner, + "pipeline": pipeline, + "exclusive": True, + "executionEngine": { + "forge": { + "project": project, + "args": args_list, + "configOverrides": { + version_key: version, + }, + }, + }, + }, + } + + manifest_json = json.dumps(fjob_manifest) + logger.info( + "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", + project, version, cluster, pipeline, + ) + + result = subprocess.run( + ["oc", "apply", "-f", "-"], + input=manifest_json, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + raise RuntimeError(f"Failed to create FournosJob: {result.stderr}") + + logger.info("FournosJob created: %s", result.stdout.strip()) + + +def run(): + """Entrypoint for the confirm-nightly phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + image_version = _read_resolved_version() + logger.info("New version available: %s", image_version) + + module = importlib.import_module(f"projects.{project}.resolvers") + verifier = module.get_verifier() + + logger.info("Using verifier: %s", type(verifier).__name__) + last_version = verifier.get_last_tested_version() + logger.info("Last tested version: '%s'", last_version or "") + + if image_version == last_version: + logger.info("Version %s already tested. No new run needed.", image_version) + print("RESULT: NO-OP") + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "result.txt").write_text("NO-OP") + return + + logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") + + _create_fournos_job(project, image_version) + + result_msg = f"FournosJob created for {project} version {image_version}" + logger.info("RESULT: %s", result_msg) + print(f"RESULT: {result_msg}") + + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "result.txt").write_text(result_msg) diff --git a/projects/core/nightly/receive_image.py b/projects/core/nightly/receive_image.py new file mode 100644 index 000000000..0bc2d3979 --- /dev/null +++ b/projects/core/nightly/receive_image.py @@ -0,0 +1,42 @@ +"""Nightly receive-image handler. + +Loads the project's resolvers module and calls get_receiver() to obtain +an ImageReceiver instance. The core never knows what registry or strategy +the project uses — it just calls get_latest_version() on whatever it gets. +""" + +from __future__ import annotations + +import importlib +import logging +import os +from pathlib import Path + +from projects.core.library import config + +logger = logging.getLogger(__name__) + + +def run(): + """Entrypoint for the receive-image phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + module = importlib.import_module(f"projects.{project}.resolvers") + receiver = module.get_receiver() + + logger.info("Using receiver: %s", type(receiver).__name__) + version = receiver.get_latest_version() + logger.info("Resolved version: %s", version) + + artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) + version_file = artifact_base / "resolved_version.txt" + version_file.write_text(version) + logger.info("Wrote version to %s", version_file) + + artifact_dir = Path(os.environ["ARTIFACT_DIR"]) + (artifact_dir / "resolved_version.txt").write_text(version) + + print(f"RESOLVED_VERSION={version}") diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index b73f7da9d..d592c8bcf 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,3 +1,10 @@ +nightly: + cluster: "smoke-cluster" + version_key: "infrastructure.mcp_gateway_version" + pipeline: "fournos-full" + preset: "demo" + owner: "ashtakb" + vaults: all: [] all-optional: diff --git a/projects/mcp_gateway/resolvers/__init__.py b/projects/mcp_gateway/resolvers/__init__.py new file mode 100644 index 000000000..a6bf3463b --- /dev/null +++ b/projects/mcp_gateway/resolvers/__init__.py @@ -0,0 +1,66 @@ +"""MCP Gateway nightly resolvers. + +Auto-discovers ImageReceiver and NightlyVerifier subclasses in this package. +To add a new receiver or verifier, just create a new file with a class that: + - Inherits from ImageReceiver or NightlyVerifier + - Defines a NAME class attribute (used for selection) + +No modifications to this file are needed. +""" + +from __future__ import annotations + +import importlib +import os +import pkgutil + +from projects.core.nightly.base_receiver import ImageReceiver +from projects.core.nightly.base_verifier import NightlyVerifier + +_receivers: dict[str, type[ImageReceiver]] = {} +_verifiers: dict[str, type[NightlyVerifier]] = {} + + +def _discover(): + """Import all modules in this package and register subclasses.""" + if _receivers or _verifiers: + return + + for _finder, module_name, _ in pkgutil.iter_modules(__path__): + module = importlib.import_module(f"{__name__}.{module_name}") + for attr in vars(module).values(): + if not isinstance(attr, type): + continue + if issubclass(attr, ImageReceiver) and attr is not ImageReceiver: + name = getattr(attr, "NAME", None) + if name: + _receivers[name] = attr + if issubclass(attr, NightlyVerifier) and attr is not NightlyVerifier: + name = getattr(attr, "NAME", None) + if name: + _verifiers[name] = attr + + +def get_receiver() -> ImageReceiver: + """Return the image receiver based on NIGHTLY_SOURCE env var.""" + _discover() + source = os.environ.get("NIGHTLY_SOURCE", "ghcr") + + cls = _receivers.get(source) + if cls is None: + raise ValueError( + f"Unknown source '{source}' for mcp_gateway. " + f"Available: {list(_receivers.keys())}" + ) + return cls() + + +def get_verifier() -> NightlyVerifier: + """Return the nightly verifier (first discovered NightlyVerifier subclass).""" + _discover() + + if not _verifiers: + raise RuntimeError("No NightlyVerifier subclass found in resolvers package") + + cls = next(iter(_verifiers.values())) + return cls() diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml new file mode 100644 index 000000000..c18e444a3 --- /dev/null +++ b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml @@ -0,0 +1,50 @@ +# CronJob to trigger the nightly pipeline for mcp_gateway (GHCR source). +# Runs daily at 2:00 AM UTC. Creates a clusterless FournosJob that +# uses the "nightly" pipeline to check for new versions on ghcr.io. +# +# Apply manually: +# oc apply -f projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nightly-mcp-gateway-ghcr + namespace: fournos-workloads + labels: + fournos.dev/nightly: "true" + fournos.dev/project: mcp-gateway + fournos.dev/source: ghcr +spec: + schedule: "0 2 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: fournos + restartPolicy: Never + containers: + - name: submit + image: bitnami/kubectl:latest + command: ["sh", "-c"] + args: + - | + cat <<'EOF' | oc apply -f - + apiVersion: fournos.dev/v1 + kind: FournosJob + metadata: + generateName: nightly-mcp-gateway- + namespace: fournos-workloads + spec: + clusterless: true + exclusive: false + pipeline: "nightly" + env: + NIGHTLY_SOURCE: "ghcr" + executionEngine: + forge: + project: "mcp_gateway" + args: [] + EOF + echo "Nightly FournosJob created for mcp_gateway (ghcr)" diff --git a/projects/mcp_gateway/resolvers/ghcr_receiver.py b/projects/mcp_gateway/resolvers/ghcr_receiver.py new file mode 100644 index 000000000..b07db825d --- /dev/null +++ b/projects/mcp_gateway/resolvers/ghcr_receiver.py @@ -0,0 +1,70 @@ +"""GHCR image receiver for mcp_gateway. + +Resolves the latest mcp-gateway commit SHA that has a published image +on ghcr.io. Checks recent commits from the repo and finds the first +one with a corresponding container image tag (sha-). + +No authentication needed for public packages (anonymous token). +""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request + +from projects.core.nightly.base_receiver import ImageReceiver + +REPO = "Kuadrant/mcp-gateway" +IMAGE = "kuadrant/mcp-gateway" +COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" +TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" +MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" + + +class GHCRReceiver(ImageReceiver): + """Find the latest mcp-gateway commit with a published ghcr.io image.""" + + NAME = "ghcr" + + def _fetch_version(self) -> str: + token = self._get_ghcr_token() + shas = self._get_recent_commits() + + for sha in shas: + if self._image_exists(sha, token): + return sha + + raise RuntimeError( + f"No commit found with a published image on ghcr.io " + f"(checked {len(shas)} recent commits)" + ) + + def _get_ghcr_token(self) -> str: + with urllib.request.urlopen(TOKEN_URL, timeout=30) as resp: + data = json.loads(resp.read()) + return data["token"] + + def _get_recent_commits(self) -> list[str]: + req = urllib.request.Request(COMMITS_URL) + with urllib.request.urlopen(req, timeout=30) as resp: + commits = json.loads(resp.read()) + return [c["sha"] for c in commits] + + def _image_exists(self, sha: str, token: str) -> bool: + tag = f"sha-{sha}" + req = urllib.request.Request( + MANIFEST_URL + tag, + headers={ + "Accept": "application/vnd.oci.image.index.v1+json, " + "application/vnd.docker.distribution.manifest.v2+json", + "Authorization": f"Bearer {token}", + }, + ) + try: + urllib.request.urlopen(req, timeout=30) + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise diff --git a/projects/mcp_gateway/resolvers/mlflow_verifier.py b/projects/mcp_gateway/resolvers/mlflow_verifier.py new file mode 100644 index 000000000..a956854cb --- /dev/null +++ b/projects/mcp_gateway/resolvers/mlflow_verifier.py @@ -0,0 +1,121 @@ +"""MLflow nightly verifier for mcp_gateway. + +Connects to the MLflow tracking server, queries runs in the project's +experiment, and extracts the version from the most recent run name. + +Uses the same secret loading mechanism as caliper's mlflow export. +""" + +from __future__ import annotations + +import logging +import os +import re +from pathlib import Path + +from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + mlflow_connection_env, + validate_mlflow_secrets, +) +from projects.core.nightly.base_verifier import NightlyVerifier + +logger = logging.getLogger(__name__) + +VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") + + +class MLflowVerifier(NightlyVerifier): + """Check MLflow experiment runs for the last tested version. + + Reads its configuration from the project's config.yaml under: + caliper.export.backend.mlflow.config.experiment + caliper.export.backend.mlflow.config.workspace + caliper.export.backend.mlflow.secrets.vault.name + caliper.export.backend.mlflow.secrets.vault.mlflow_secret + """ + + NAME = "mlflow" + + def __init__(self): + from projects.core.library import config + + self.experiment = config.project.get_config( + "caliper.export.backend.mlflow.config.experiment" + ) + self.workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace" + ) + vault_cfg = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault" + ) + self.secret_name = vault_cfg["name"] + self.secret_file = vault_cfg["mlflow_secret"] + + def _fetch_last_version(self) -> str: + connection = self._load_connection() + + with mlflow_connection_env(connection): + return self._query_latest_run() + + def _load_connection(self) -> dict: + secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) + secret_path = secrets_dir / self.secret_name / self.secret_file + + if not secret_path.exists(): + logger.warning( + "MLflow secret not found at expected path, assuming no previous run" + ) + return {} + + data = load_mlflow_secrets_yaml(secret_path) + validate_mlflow_secrets(data) + return data + + def _query_latest_run(self) -> str: + import mlflow + + if self.workspace: + os.environ["MLFLOW_WORKSPACE"] = self.workspace + + client = mlflow.tracking.MlflowClient() + + experiment = client.get_experiment_by_name(self.experiment) + if experiment is None: + logger.warning("Experiment '%s' not found in MLflow", self.experiment) + return "" + + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + order_by=["start_time DESC"], + max_results=1, + ) + + if not runs: + logger.info("No runs found in experiment '%s'", self.experiment) + return "" + + latest_run = runs[0] + run_name = latest_run.info.run_name or "" + logger.info("Latest MLflow run: %s", run_name) + + return self._extract_version(run_name) + + def _extract_version(self, run_name: str) -> str: + """Extract the version from a run name like: + forge-mcp-gateway-s150-u500-vsha--20260714-224646 + + Returns the raw SHA (without the 'sha-' prefix) to match + what the receiver returns. + """ + match = VERSION_PATTERN.search(run_name) + if not match: + logger.warning("Could not extract version from run name: %s", run_name) + return "" + + version = match.group(1) + + if version.startswith("sha-"): + return version[4:] + + return version From 625cf603d38450190ae09d3c431060cd9561cd1c Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 13:18:52 +0300 Subject: [PATCH 0136/1072] fix: use openshift origin-cli image for oc in cronjob Co-authored-by: Cursor --- projects/mcp_gateway/resolvers/cronjob-ghcr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml index c18e444a3..2f6a9b472 100644 --- a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml @@ -26,7 +26,7 @@ spec: restartPolicy: Never containers: - name: submit - image: bitnami/kubectl:latest + image: quay.io/openshift/origin-cli:latest command: ["sh", "-c"] args: - | From e16dc568413eb088a4cfcaa9e4b999cb321ae544 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 15:43:42 +0300 Subject: [PATCH 0137/1072] refactor: collapse nightly pipeline into single task with notifications Address PR review comments: - Merge two pipeline tasks (receive-image, confirm-nightly) into single "nightly" task for simplicity; error visibility via notification files - Remove retry from base ABCs, keep it in concrete implementations - Reuse fournos_launcher submit_and_wait with wait=False + EarlyReturn - Rename resolvers/ to nightly_resolvers/ - Use env module instead of raw os.environ for artifact paths - Make should_init_vaults configurable via vaults.skip_phases in config - Use quay.io/openshift/origin-cli image, fix namespace to psap-automation - Fix list_vaults() to skip the skip_phases key Co-authored-by: Cursor --- .../base/workflows/pipeline-nightly.yaml | 32 +---- projects/agentic_tools/ci_base.py | 21 +-- projects/core/nightly/__init__.py | 14 +- projects/core/nightly/base_receiver.py | 27 +--- projects/core/nightly/base_verifier.py | 27 +--- projects/core/nightly/confirm.py | 131 ------------------ projects/core/nightly/handler.py | 117 ++++++++++++++++ projects/core/nightly/receive_image.py | 42 ------ .../toolbox/submit_and_wait/main.py | 5 + .../__init__.py | 3 +- .../cronjob-ghcr.yaml | 4 +- .../ghcr_receiver.py | 27 +++- .../mlflow_verifier.py | 39 ++++-- .../mcp_gateway/orchestration/config.yaml | 2 + 14 files changed, 205 insertions(+), 286 deletions(-) delete mode 100644 projects/core/nightly/confirm.py create mode 100644 projects/core/nightly/handler.py delete mode 100644 projects/core/nightly/receive_image.py rename projects/mcp_gateway/{resolvers => nightly_resolvers}/__init__.py (94%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/cronjob-ghcr.yaml (93%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/ghcr_receiver.py (70%) rename projects/mcp_gateway/{resolvers => nightly_resolvers}/mlflow_verifier.py (73%) diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml index 18f729de1..a3e7a38ab 100644 --- a/fournos/gitops/base/workflows/pipeline-nightly.yaml +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -1,11 +1,11 @@ # Nightly pipeline: clusterless job that checks for new image versions # and conditionally triggers a full test FournosJob. # -# Flow: receive-image (resolve latest version) → confirm-nightly (compare with S3, create FournosJob if new) +# Single step that resolves the latest version, compares against the last +# tested version, and creates a FournosJob if a new version is detected. +# Error visibility is handled via notification files in the artifact directory. # # This pipeline runs as a clusterless FournosJob (no target cluster needed). -# Both tasks use the standard forge-step, dispatching to phases registered -# in the project's ci.py (backed by shared handlers in projects/core/nightly/). apiVersion: tekton.dev/v1 kind: Pipeline metadata: @@ -23,7 +23,7 @@ spec: - name: kubeconfig-secret type: string tasks: - - name: 00-receive-image + - name: 00-nightly taskRef: name: forge-step kind: Task @@ -38,26 +38,6 @@ spec: - name: kubeconfig-secret value: "$(params.kubeconfig-secret)" - name: job-step - value: receive-image + value: nightly - name: job-step-name - value: 00__receive-image - - - name: 01-confirm-nightly - runAfter: [00-receive-image] - taskRef: - name: forge-step - kind: Task - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: confirm-nightly - - name: job-step-name - value: 01__confirm-nightly + value: 00__nightly diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index 853848660..3c078ab70 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -127,14 +127,19 @@ def list_vaults(self) -> list[str]: if isinstance(vault_config, list): return vault_config all_vaults: list[str] = [] - for _category, vaults in vault_config.items(): + for category, vaults in vault_config.items(): + if category == "skip_phases": + continue if isinstance(vaults, list): all_vaults.extend(vaults) return list(dict.fromkeys(all_vaults)) def should_init_vaults(self, phase: str) -> bool: - """Return False to skip vault init for certain phases.""" - return phase not in ("resolve-fournos-config", "receive-image", "confirm-nightly") + """Return False to skip vault init for resolve and phases listed in vaults.skip_phases.""" + if phase == "resolve-fournos-config": + return False + skip_phases = config.project.get_config("vaults.skip_phases", [], print=False) + return phase not in skip_phases def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. @@ -177,13 +182,9 @@ class path like ``projects.foo.notifications.MyProvider``) and # ------------------------------------------------------------------ _NIGHTLY_PHASES: dict[str, PhaseSpec] = { - "receive-image": PhaseSpec( - func="projects.core.nightly.receive_image:run", - help="Nightly: resolve latest available image version.", - ), - "confirm-nightly": PhaseSpec( - func="projects.core.nightly.confirm:run", - help="Nightly: check S3 for last run, create FournosJob if new version.", + "nightly": PhaseSpec( + func="projects.core.nightly.handler:run", + help="Nightly: check for new image version and trigger test if needed.", ), } diff --git a/projects/core/nightly/__init__.py b/projects/core/nightly/__init__.py index 183c607a2..c0cdba66f 100644 --- a/projects/core/nightly/__init__.py +++ b/projects/core/nightly/__init__.py @@ -2,21 +2,19 @@ Architecture: Core (this package) defines abstract interfaces and the orchestration logic. - Each project implements concrete receivers/verifiers in its resolvers/ package + Each project implements concrete receivers/verifiers in its nightly_resolvers/ package and exposes them via get_receiver() and get_verifier() factory functions. - The core handlers import the project's resolvers module at runtime and call + The core handler imports the project's nightly_resolvers module at runtime and calls those factories — pure polymorphism, no knowledge of concrete implementations. Interfaces: - ImageReceiver: fetches the latest version from a registry - NightlyVerifier: checks if a version was already tested -Handlers: - - receive_image.run(): calls project's get_receiver().get_latest_version() - - confirm.run(): calls project's get_verifier().get_last_tested_version() +Handler: + - handler.run(): single phase that resolves version, compares, and creates FournosJob -Projects register these as phases in ci_base: - "receive-image" -> projects.core.nightly.receive_image:run - "confirm-nightly" -> projects.core.nightly.confirm:run +Projects register this as a phase in ci_base: + "nightly" -> projects.core.nightly.handler:run """ diff --git a/projects/core/nightly/base_receiver.py b/projects/core/nightly/base_receiver.py index ba93c82c0..2e6e70ab7 100644 --- a/projects/core/nightly/base_receiver.py +++ b/projects/core/nightly/base_receiver.py @@ -6,41 +6,16 @@ from __future__ import annotations -import logging -import time from abc import ABC, abstractmethod -logger = logging.getLogger(__name__) - -MAX_ATTEMPTS = 3 -RETRY_DELAY_SECONDS = 10 - class ImageReceiver(ABC): """Fetches the latest available version from a container registry.""" @abstractmethod - def _fetch_version(self) -> str: + def get_latest_version(self) -> str: """Query the registry and return the latest version string. Raises on failure (network, auth, no tags found, etc.). """ ... - - def get_latest_version(self) -> str: - """Call _fetch_version with retry on transient failures.""" - last_error = None - for attempt in range(1, MAX_ATTEMPTS + 1): - try: - return self._fetch_version() - except Exception as e: - last_error = e - if attempt < MAX_ATTEMPTS: - logger.warning( - "Receiver attempt %d/%d failed: %s. Retrying in %ds...", - attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, - ) - time.sleep(RETRY_DELAY_SECONDS) - raise RuntimeError( - f"Receiver failed after {MAX_ATTEMPTS} attempts" - ) from last_error diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py index 8d1212aa1..a0c644d6d 100644 --- a/projects/core/nightly/base_verifier.py +++ b/projects/core/nightly/base_verifier.py @@ -6,41 +6,16 @@ from __future__ import annotations -import logging -import time from abc import ABC, abstractmethod -logger = logging.getLogger(__name__) - -MAX_ATTEMPTS = 3 -RETRY_DELAY_SECONDS = 10 - class NightlyVerifier(ABC): """Checks historical run data to determine if a version was already tested.""" @abstractmethod - def _fetch_last_version(self) -> str: + def get_last_tested_version(self) -> str: """Return the version string of the last successful run. Returns empty string if no previous run exists. """ ... - - def get_last_tested_version(self) -> str: - """Call _fetch_last_version with retry on transient failures.""" - last_error = None - for attempt in range(1, MAX_ATTEMPTS + 1): - try: - return self._fetch_last_version() - except Exception as e: - last_error = e - if attempt < MAX_ATTEMPTS: - logger.warning( - "Verifier attempt %d/%d failed: %s. Retrying in %ds...", - attempt, MAX_ATTEMPTS, e, RETRY_DELAY_SECONDS, - ) - time.sleep(RETRY_DELAY_SECONDS) - raise RuntimeError( - f"Verifier failed after {MAX_ATTEMPTS} attempts" - ) from last_error diff --git a/projects/core/nightly/confirm.py b/projects/core/nightly/confirm.py deleted file mode 100644 index b677b0176..000000000 --- a/projects/core/nightly/confirm.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Nightly confirm handler. - -Loads the project's resolvers module to get a NightlyVerifier instance. -Compares the resolved version (from receive-image step) against what -the verifier reports as the last tested version. Creates a FournosJob -if a new version is detected. - -The core never knows how or where history is stored — it just calls -get_last_tested_version() on whatever the project provides. -""" - -from __future__ import annotations - -import importlib -import json -import logging -import os -import subprocess -from pathlib import Path - -from projects.core.library import config - -logger = logging.getLogger(__name__) - - -def _read_resolved_version() -> str: - """Read the version written by the receive-image step.""" - artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) - version_file = artifact_base / "resolved_version.txt" - - if not version_file.exists(): - raise FileNotFoundError( - f"resolved_version.txt not found at {version_file}. " - "Did the receive-image step run successfully?" - ) - - version = version_file.read_text().strip() - if not version: - raise RuntimeError("resolved_version.txt is empty") - return version - - -def _create_fournos_job(project: str, version: str) -> None: - """Create a FournosJob for the full test run.""" - namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") - cluster = config.project.get_config("nightly.cluster") - version_key = config.project.get_config("nightly.version_key") - pipeline = config.project.get_config("nightly.pipeline") - preset = config.project.get_config("nightly.preset", "") - owner = config.project.get_config("nightly.owner", "nightly-pipeline") - - args_list = [preset] if preset else [] - - fjob_manifest = { - "apiVersion": "fournos.dev/v1", - "kind": "FournosJob", - "metadata": { - "generateName": f"nightly-{project.replace('_', '-')}-", - "namespace": namespace, - }, - "spec": { - "cluster": cluster, - "displayName": f"nightly {project} {version}", - "owner": owner, - "pipeline": pipeline, - "exclusive": True, - "executionEngine": { - "forge": { - "project": project, - "args": args_list, - "configOverrides": { - version_key: version, - }, - }, - }, - }, - } - - manifest_json = json.dumps(fjob_manifest) - logger.info( - "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", - project, version, cluster, pipeline, - ) - - result = subprocess.run( - ["oc", "apply", "-f", "-"], - input=manifest_json, - capture_output=True, - text=True, - ) - - if result.returncode != 0: - raise RuntimeError(f"Failed to create FournosJob: {result.stderr}") - - logger.info("FournosJob created: %s", result.stdout.strip()) - - -def run(): - """Entrypoint for the confirm-nightly phase.""" - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - project = config.project.get_config("project.name") - logger.info("Project: %s", project) - - image_version = _read_resolved_version() - logger.info("New version available: %s", image_version) - - module = importlib.import_module(f"projects.{project}.resolvers") - verifier = module.get_verifier() - - logger.info("Using verifier: %s", type(verifier).__name__) - last_version = verifier.get_last_tested_version() - logger.info("Last tested version: '%s'", last_version or "") - - if image_version == last_version: - logger.info("Version %s already tested. No new run needed.", image_version) - print("RESULT: NO-OP") - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "result.txt").write_text("NO-OP") - return - - logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") - - _create_fournos_job(project, image_version) - - result_msg = f"FournosJob created for {project} version {image_version}" - logger.info("RESULT: %s", result_msg) - print(f"RESULT: {result_msg}") - - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "result.txt").write_text(result_msg) diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py new file mode 100644 index 000000000..2ea3e3286 --- /dev/null +++ b/projects/core/nightly/handler.py @@ -0,0 +1,117 @@ +"""Nightly pipeline handler — single phase that orchestrates: + +1. Resolve the latest available image version (via project's ImageReceiver) +2. Compare against last tested version (via project's NightlyVerifier) +3. Create a FournosJob if a new version is detected + +Error visibility is provided via notification files in the CI metadata directory. +""" + +from __future__ import annotations + +import importlib +import logging +import os + +from projects.core.library import ci as ci_lib +from projects.core.library import config, env + +logger = logging.getLogger(__name__) + + +def _receive_image(project: str) -> str: + """Resolve the latest available version from the project's receiver.""" + module = importlib.import_module(f"projects.{project}.nightly_resolvers") + receiver = module.get_receiver() + + logger.info("Using receiver: %s", type(receiver).__name__) + version = receiver.get_latest_version() + logger.info("Resolved version: %s", version) + + version_file = env.BASE_ARTIFACT_DIR / "resolved_version.txt" + version_file.write_text(version) + + (env.ARTIFACT_DIR / "resolved_version.txt").write_text(version) + return version + + +def _confirm_nightly(project: str, image_version: str) -> None: + """Compare resolved version against last tested, create FournosJob if new.""" + module = importlib.import_module(f"projects.{project}.nightly_resolvers") + verifier = module.get_verifier() + + logger.info("Using verifier: %s", type(verifier).__name__) + last_version = verifier.get_last_tested_version() + logger.info("Last tested version: '%s'", last_version or "") + + if image_version == last_version: + logger.info("Version %s already tested. No new run needed.", image_version) + ci_lib.add_notification_file( + "nightly-noop", f"Version {image_version} already tested — no-op." + ) + (env.ARTIFACT_DIR / "result.txt").write_text("NO-OP") + return + + logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") + _create_fournos_job(project, image_version) + + result_msg = f"FournosJob created for {project} version {image_version}" + ci_lib.add_notification_file("nightly-triggered", result_msg) + (env.ARTIFACT_DIR / "result.txt").write_text(result_msg) + + +def _create_fournos_job(project: str, version: str) -> None: + """Create a FournosJob for the full test run using the fournos_launcher toolbox.""" + from projects.fournos_launcher.toolbox.submit_and_wait.main import run as submit_and_wait + + namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") + cluster = config.project.get_config("nightly.cluster") + version_key = config.project.get_config("nightly.version_key") + pipeline = config.project.get_config("nightly.pipeline") + preset = config.project.get_config("nightly.preset", "") + owner = config.project.get_config("nightly.owner", "nightly-pipeline") + + args_list = [preset] if preset else [] + + logger.info( + "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", + project, + version, + cluster, + pipeline, + ) + + submit_and_wait( + project, + cluster_name=cluster, + args=args_list, + variables_overrides={version_key: version}, + namespace=namespace, + owner=owner, + display_name=f"nightly {project} {version}", + pipeline_name=pipeline, + exclusive=True, + wait=False, + ) + + +def run(): + """Entrypoint for the nightly phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + logger.info("Project: %s", project) + + try: + image_version = _receive_image(project) + except Exception as e: + logger.error("receive-image failed: %s", e) + ci_lib.add_notification_file("nightly-receive-image-failed", f"FATAL: {e}") + raise + + try: + _confirm_nightly(project, image_version) + except Exception as e: + logger.error("confirm-nightly failed: %s", e) + ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") + raise diff --git a/projects/core/nightly/receive_image.py b/projects/core/nightly/receive_image.py deleted file mode 100644 index 0bc2d3979..000000000 --- a/projects/core/nightly/receive_image.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Nightly receive-image handler. - -Loads the project's resolvers module and calls get_receiver() to obtain -an ImageReceiver instance. The core never knows what registry or strategy -the project uses — it just calls get_latest_version() on whatever it gets. -""" - -from __future__ import annotations - -import importlib -import logging -import os -from pathlib import Path - -from projects.core.library import config - -logger = logging.getLogger(__name__) - - -def run(): - """Entrypoint for the receive-image phase.""" - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - project = config.project.get_config("project.name") - logger.info("Project: %s", project) - - module = importlib.import_module(f"projects.{project}.resolvers") - receiver = module.get_receiver() - - logger.info("Using receiver: %s", type(receiver).__name__) - version = receiver.get_latest_version() - logger.info("Resolved version: %s", version) - - artifact_base = Path(os.environ["ARTIFACT_BASE_DIR"]) - version_file = artifact_base / "resolved_version.txt" - version_file.write_text(version) - logger.info("Wrote version to %s", version_file) - - artifact_dir = Path(os.environ["ARTIFACT_DIR"]) - (artifact_dir / "resolved_version.txt").write_text(version) - - print(f"RESOLVED_VERSION={version}") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 40e4917b6..7272fae7f 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -20,6 +20,7 @@ task, template, ) +from projects.core.dsl.control_flow import EarlyReturn from projects.core.dsl.utils.k8s import is_valid_k8s_name, sanitize_k8s_name from projects.core.library import env as env_mod @@ -64,6 +65,7 @@ def run( gpu_count: int = None, gpu_type: str = None, clusterless: bool = False, + wait: bool = True, ): """ Submit a FOURNOS job and wait for completion @@ -208,6 +210,9 @@ def submit_fournos_job(args, ctx): # Apply the job manifest (will raise CalledProcessError with full details on failure) shell.run(f"oc apply -f {ctx.manifest_file}") + if not args.wait: + return EarlyReturn(f"Submitted FournosJob: {ctx.final_job_name} (wait=False)") + return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" diff --git a/projects/mcp_gateway/resolvers/__init__.py b/projects/mcp_gateway/nightly_resolvers/__init__.py similarity index 94% rename from projects/mcp_gateway/resolvers/__init__.py rename to projects/mcp_gateway/nightly_resolvers/__init__.py index a6bf3463b..27c4f1e56 100644 --- a/projects/mcp_gateway/resolvers/__init__.py +++ b/projects/mcp_gateway/nightly_resolvers/__init__.py @@ -49,8 +49,7 @@ def get_receiver() -> ImageReceiver: cls = _receivers.get(source) if cls is None: raise ValueError( - f"Unknown source '{source}' for mcp_gateway. " - f"Available: {list(_receivers.keys())}" + f"Unknown source '{source}' for mcp_gateway. Available: {list(_receivers.keys())}" ) return cls() diff --git a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml similarity index 93% rename from projects/mcp_gateway/resolvers/cronjob-ghcr.yaml rename to projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml index 2f6a9b472..bf053c29a 100644 --- a/projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml @@ -3,12 +3,12 @@ # uses the "nightly" pipeline to check for new versions on ghcr.io. # # Apply manually: -# oc apply -f projects/mcp_gateway/resolvers/cronjob-ghcr.yaml +# oc apply -f projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml apiVersion: batch/v1 kind: CronJob metadata: name: nightly-mcp-gateway-ghcr - namespace: fournos-workloads + namespace: psap-automation labels: fournos.dev/nightly: "true" fournos.dev/project: mcp-gateway diff --git a/projects/mcp_gateway/resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py similarity index 70% rename from projects/mcp_gateway/resolvers/ghcr_receiver.py rename to projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index b07db825d..949de372f 100644 --- a/projects/mcp_gateway/resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -10,23 +10,48 @@ from __future__ import annotations import json +import logging +import time import urllib.error import urllib.request from projects.core.nightly.base_receiver import ImageReceiver +logger = logging.getLogger(__name__) + REPO = "Kuadrant/mcp-gateway" IMAGE = "kuadrant/mcp-gateway" COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + class GHCRReceiver(ImageReceiver): """Find the latest mcp-gateway commit with a published ghcr.io image.""" NAME = "ghcr" + def get_latest_version(self) -> str: + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_version() + except urllib.error.URLError as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "GHCR attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"GHCR receiver failed after {MAX_ATTEMPTS} attempts") from last_error + def _fetch_version(self) -> str: token = self._get_ghcr_token() shas = self._get_recent_commits() @@ -57,7 +82,7 @@ def _image_exists(self, sha: str, token: str) -> bool: MANIFEST_URL + tag, headers={ "Accept": "application/vnd.oci.image.index.v1+json, " - "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", "Authorization": f"Bearer {token}", }, ) diff --git a/projects/mcp_gateway/resolvers/mlflow_verifier.py b/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py similarity index 73% rename from projects/mcp_gateway/resolvers/mlflow_verifier.py rename to projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py index a956854cb..2e9628a77 100644 --- a/projects/mcp_gateway/resolvers/mlflow_verifier.py +++ b/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py @@ -11,6 +11,7 @@ import logging import os import re +import time from pathlib import Path from projects.caliper.engine.file_export.mlflow_secrets import ( @@ -24,6 +25,9 @@ VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 15 + class MLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. @@ -43,29 +47,40 @@ def __init__(self): self.experiment = config.project.get_config( "caliper.export.backend.mlflow.config.experiment" ) - self.workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace" - ) - vault_cfg = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault" - ) + self.workspace = config.project.get_config("caliper.export.backend.mlflow.config.workspace") + vault_cfg = config.project.get_config("caliper.export.backend.mlflow.secrets.vault") self.secret_name = vault_cfg["name"] self.secret_file = vault_cfg["mlflow_secret"] - def _fetch_last_version(self) -> str: + def get_last_tested_version(self) -> str: connection = self._load_connection() + if not connection: + return "" - with mlflow_connection_env(connection): - return self._query_latest_run() + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + with mlflow_connection_env(connection): + return self._query_latest_run() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "MLflow attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"MLflow verifier failed after {MAX_ATTEMPTS} attempts") from last_error def _load_connection(self) -> dict: secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) secret_path = secrets_dir / self.secret_name / self.secret_file if not secret_path.exists(): - logger.warning( - "MLflow secret not found at expected path, assuming no previous run" - ) + logger.warning("MLflow secret not found at expected path, assuming no previous run") return {} data = load_mlflow_secrets_yaml(secret_path) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index d592c8bcf..ee211ebf1 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -12,6 +12,8 @@ vaults: - psap-forge-mlflow-export prepare: [] test: [] + skip_phases: + - nightly notifications: slack: From 318657bf00a492ed0397b730be016fc7b75ec154 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 18:47:56 +0300 Subject: [PATCH 0138/1072] fix: use correct cluster name for nightly job target Co-authored-by: Cursor --- projects/mcp_gateway/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index ee211ebf1..2f469520d 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,5 +1,5 @@ nightly: - cluster: "smoke-cluster" + cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" pipeline: "fournos-full" preset: "demo" From 9370ef2f1ff4b92afb022cd3add1798000a3c40e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 18:54:16 +0300 Subject: [PATCH 0139/1072] fix: correct pipeline name to forge-full Co-authored-by: Cursor --- projects/mcp_gateway/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 2f469520d..8abbba249 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,7 +1,7 @@ nightly: cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" - pipeline: "fournos-full" + pipeline: "forge-full" preset: "demo" owner: "ashtakb" From fdc49f1c13aa49e2171e57adcf0d249308857a73 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 20 Jul 2026 19:33:45 +0300 Subject: [PATCH 0140/1072] fix: rename _mesh_ready_spec to propagate across DSL tasks The DSL runtime skips underscore-prefixed context attributes when propagating between tasks, causing wait_service_mesh_ready to never see the readiness spec set by install_service_mesh_instance. Co-authored-by: Cursor --- projects/mcp_gateway/toolbox/install_platform/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index fcf5c5d5c..6fdc75046 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -115,7 +115,7 @@ def install_service_mesh_instance(args, ctx): if "wait_for_ready" in step: spec = step["wait_for_ready"] - ctx._mesh_ready_spec = spec + ctx.mesh_ready_spec = spec return (False, "Service Mesh instance applied, readiness checked in next task") return f"Service Mesh instance applied from {kustomize_path}" @@ -125,7 +125,7 @@ def install_service_mesh_instance(args, ctx): @task def wait_service_mesh_ready(args, ctx): """Wait for Istio instance to report Ready condition""" - spec = getattr(ctx, "_mesh_ready_spec", None) + spec = getattr(ctx, "mesh_ready_spec", None) if not spec: return "No readiness spec, skipping" From b983a1d290dff06192b481146b1b0a1fecc2af7b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 14:01:50 -0400 Subject: [PATCH 0141/1072] Add Slack failure alert when RHAIIS pipeline crashes Sends a webhook notification with job metadata (model, TP/DP, version, cluster, workloads, error) so failures are visible without checking the Fournos dashboard. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 58 ++++++++++-- projects/rhaiis/postprocess/regression.py | 99 +++++++++++++++++++++ 2 files changed, 150 insertions(+), 7 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 180cd46e7..942d127de 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import os +import traceback import uuid as _uuid_mod from projects.core.library import env @@ -12,6 +14,44 @@ _K8S_NAME_MAX = 63 +def _send_pipeline_failure_alert( + exc: Exception, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert when the pipeline fails.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=traceback.format_exception_only(type(exc), exc)[-1].strip(), + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline failure alert", exc_info=True) + + def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -43,13 +83,17 @@ def do_test( namespace: str, deployment_name: str | None = None, ) -> int: - with env.NextArtifactDir("testing"): - return _run_test( - model_key=model_key, - workload_keys=workload_keys, - namespace=namespace, - deployment_name=deployment_name, - ) + try: + with env.NextArtifactDir("testing"): + return _run_test( + model_key=model_key, + workload_keys=workload_keys, + namespace=namespace, + deployment_name=deployment_name, + ) + except Exception as exc: + _send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) + raise def _run_test( diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index d992e859d..87273f854 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -222,3 +222,102 @@ def send_regression_notification( dry_run=dry_run, notification_vault=notification_vault, ) + + +def send_failure_notification( + *, + error: str, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + webhook_vault: str | None = None, + notification_vault: str | None = None, + dry_run: bool = False, + tp: str = "", + dp: str = "", + version: str = "", + workload_keys: list[str] | None = None, + cluster: str = "", +) -> bool: + """Send a Slack alert when the RHAIIS pipeline fails. + + Args: + error: Error message or traceback summary + model: Model name for display + accelerator: Accelerator name for display + job_id: FournosJob name + slack_user: Slack user ID to @-mention + webhook_vault: Vault containing slack-webhook-url file + notification_vault: Fallback vault for Bot-Token Slack credentials + dry_run: Log only, don't send + tp: Tensor parallelism size + dp: Data parallelism size + version: vLLM / RHAIIS version string + workload_keys: List of workload profile keys + cluster: Cluster name the job ran on + + Returns: + True if notification sent successfully + """ + from projects.core.notifications.send import ( + _get_webhook_url, + _send_via_webhook, + send_notification, + ) + + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + + profiles_line = "" + if workload_keys: + profiles_line = f"*Workloads:* {', '.join(workload_keys)}\n" + + cluster_line = f"*Cluster:* {cluster}\n" if cluster else "" + version_line = f"*Version:* {version}\n" if version else "" + + error_text = error if len(error) <= 500 else error[:500] + "..." + + message = ( + f":x: *RHAIIS Pipeline Failed*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"{version_line}" + f"{cluster_line}" + f"{profiles_line}" + f"*Error:*\n```{error_text}```" + ) + + if dry_run: + logger.info("DRY RUN failure notification:\n%s", message) + return True + + webhook_url = None + if webhook_vault: + webhook_url = _get_webhook_url(webhook_vault) + + if webhook_url: + blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] + return _send_via_webhook(webhook_url, blocks) + + return send_notification( + message, + github=False, + slack=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) From 3948bd1c6b29980e781546c592c79ad039559ef4 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 15:17:55 -0400 Subject: [PATCH 0142/1072] Append FJOB suffix to deployment name for parallel job isolation Each FournosJob now gets a unique vLLM deployment name derived from the FJOB_NAME suffix, allowing multiple jobs to run in parallel on the same namespace without colliding on the same InferenceService. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 942d127de..df526a651 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -113,7 +113,10 @@ def _run_test( benchmark_cfg = runtime_config.get_benchmark_config() if not deployment_name: - deployment_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) + base_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) + fjob = os.environ.get("FJOB_NAME", "") + suffix = fjob.rsplit("-", 1)[-1] if fjob else "" + deployment_name = f"{base_name}-{suffix}" if suffix else base_name vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() From d4c7600c80a31676097079625c85a70b00165718 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 16:43:43 -0400 Subject: [PATCH 0143/1072] Switch RHAIIS Slack notifications to topsail bot on psap-rhaiis-alerts channel - Use topsail bot token instead of webhook for sending Slack notifications - Route RHAIIS notifications to dedicated psap-rhaiis-alerts channel - Add pipeline-level failure detection in post_cleanup for infra errors (e.g. image pull timeout) - Fix UnboundLocalError for 'os' caused by redundant import in _run_test Co-authored-by: Cursor --- projects/rhaiis/orchestration/ci.py | 67 +++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 4 +- projects/rhaiis/postprocess/regression.py | 74 +++++++++------------ 3 files changed, 100 insertions(+), 45 deletions(-) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 6b50a96e2..25ea42404 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import logging +import os import types +from pathlib import Path import click import prepare_rhaiis @@ -16,6 +18,70 @@ logger = logging.getLogger(__name__) +def _check_pipeline_failure_and_notify() -> None: + """Detect early pipeline failures (e.g. image pull errors) and send a Slack alert. + + Runs in the post-cleanup finally step. Checks whether prior steps + produced FAILURE artifacts or the test step was skipped entirely. + """ + try: + base_dir = Path(os.environ.get("ARTIFACT_BASE_DIR", "")) + if not base_dir.is_dir(): + return + + test_dir = base_dir / "03__test" + test_ran = test_dir.exists() + + # Skip steps already handled by do_test's exception handler + failure_files = sorted( + f for f in base_dir.glob("*/FAILURE") + if f.parent.name != "03__test" + ) + + if not failure_files and test_ran: + return + + errors = [] + for f in failure_files: + step_name = f.parent.name + content = f.read_text().strip() + summary = content[:300] if content else "unknown error" + errors.append(f"[{step_name}] {summary}") + + if not test_ran and not errors: + errors.append("Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)") + + error_text = "\n".join(errors) + + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_key = _cfg.project.get_config("tests.rhaiis.model_key", "unknown") + try: + model_cfg = runtime_config.get_model(model_key) + model_name = model_cfg.get("hf_model_id", model_key) + except Exception: + model_name = model_key + + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() + + send_failure_notification( + error=error_text, + model=model_name, + accelerator=accelerator_key, + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + webhook_vault="psap-forge-notifications", + version=_cfg.project.get_config("tests.rhaiis.version", ""), + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to check/send pipeline failure notification", exc_info=True) + + def list_vaults() -> list[str]: test_rhaiis.init() return runtime_config.get_vaults() @@ -85,6 +151,7 @@ def pre_cleanup(ctx): @ci_lib.safe_ci_entrypoint def post_cleanup(ctx): """Post-cleanup phase - Clean up resources after test.""" + _check_pipeline_failure_and_notify() return prepare_rhaiis.cleanup() diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index df526a651..ce17b2984 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -136,7 +136,7 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) - import os, subprocess + import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: @@ -618,7 +618,6 @@ def _run_standalone_analysis( except Exception: logger.warning("Standalone analysis failed", exc_info=True) finally: - import os if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) if current_csv_path and os.path.exists(current_csv_path): @@ -705,7 +704,6 @@ def _run_regression_check( except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) finally: - import os if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 87273f854..2c70abc5b 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -5,8 +5,38 @@ import logging import re +import projects.core.notifications.slack.api as slack_api +from projects.core.library import vault + logger = logging.getLogger(__name__) + +RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" + + +def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None, channel_id: str | None = None) -> bool: + """Send a Slack message using the topsail bot token from the forge notifications vault.""" + vault_name = notification_vault or "psap-forge-notifications" + try: + token_path = vault.get_vault_content_path(vault_name, "topsail-bot.slack-token") + except Exception: + logger.warning("Cannot resolve topsail bot token from vault %s", vault_name) + return False + + if not token_path or not token_path.exists(): + logger.warning("topsail-bot.slack-token not found in vault %s", vault_name) + return False + + token = token_path.read_text().strip() + client = slack_api.init_client(token) + if not client: + logger.error("Failed to init Slack client with topsail bot token") + return False + + _, ok = slack_api.send_message(client, message=message, channel_id=channel_id) + return ok + + PROFILE_MAP = { (1000, 1000): "profile1", (512, 2048): "profile2", @@ -105,12 +135,6 @@ def send_regression_notification( Returns: True if notification sent successfully """ - from projects.core.notifications.send import ( - _get_webhook_url, - _send_via_webhook, - send_notification, - ) - status = analysis_result.get("status") if status == "skipped": reason = analysis_result.get("reason", "unknown") @@ -207,21 +231,7 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) + return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) def send_failure_notification( @@ -260,12 +270,6 @@ def send_failure_notification( Returns: True if notification sent successfully """ - from projects.core.notifications.send import ( - _get_webhook_url, - _send_via_webhook, - send_notification, - ) - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): user_line = f"*Triggered by:* <@{slack_user}>\n" elif slack_user: @@ -306,18 +310,4 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - webhook_url = None - if webhook_vault: - webhook_url = _get_webhook_url(webhook_vault) - - if webhook_url: - blocks = [{"type": "section", "text": {"type": "mrkdwn", "text": message}}] - return _send_via_webhook(webhook_url, blocks) - - return send_notification( - message, - github=False, - slack=True, - dry_run=dry_run, - notification_vault=notification_vault, - ) + return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) From 7850ad42a34612c43efbaa9b31272139fadc6d12 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 22:46:39 -0400 Subject: [PATCH 0144/1072] Truncate deployment name to respect KServe 63-char hostname limit Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ce17b2984..a1e264258 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -116,7 +116,14 @@ def _run_test( base_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) fjob = os.environ.get("FJOB_NAME", "") suffix = fjob.rsplit("-", 1)[-1] if fjob else "" - deployment_name = f"{base_name}-{suffix}" if suffix else base_name + if suffix: + # KServe hostname: {name}-predictor-{namespace} must be <= 63 chars + ns = runtime_config.get_namespace() + max_name_len = 63 - len(f"-predictor-{ns}") + full_name = f"{base_name}-{suffix}" + deployment_name = full_name[:max_name_len].rstrip("-") + else: + deployment_name = base_name vllm_image = runtime_config.get_vllm_image(accelerator) vllm_defaults = runtime_config.get_vllm_defaults() From e30e0ebf1266d1e0a1bdbfc9a7d344239ebf1b81 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 20 Jul 2026 23:10:18 -0400 Subject: [PATCH 0145/1072] Fix deployment name truncation to preserve unique FJOB suffix Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index a1e264258..709a03f91 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -119,9 +119,9 @@ def _run_test( if suffix: # KServe hostname: {name}-predictor-{namespace} must be <= 63 chars ns = runtime_config.get_namespace() - max_name_len = 63 - len(f"-predictor-{ns}") - full_name = f"{base_name}-{suffix}" - deployment_name = full_name[:max_name_len].rstrip("-") + max_base = 63 - len(f"-predictor-{ns}") - len(suffix) - 1 + truncated_base = base_name[:max_base].rstrip("-") + deployment_name = f"{truncated_base}-{suffix}" else: deployment_name = base_name From 0119b54960964252ad867d646efcc35d88735ed8 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Tue, 21 Jul 2026 10:42:17 +0300 Subject: [PATCH 0146/1072] Add AGENTS.md with security guidelines for secret handling --- AGENTS.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..6da05ab1d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,73 @@ +# FORGE CI Framework — Agent Instructions + +## Project Overview + +FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on OpenShift clusters. +It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging. + +Key paths: +- `projects/` — individual project toolboxes (install scripts, test harnesses) +- `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers) +- `fournos/gitops/` — Tekton pipelines and GitOps base workflows +- `docs/` — project documentation + +## Security: Secret Handling (CRITICAL) + +**NEVER write secrets or sensitive data to artifact directories, logs, or files.** + +### Rules + +1. **Never persist secrets to `ARTIFACT_DIR`.** + The `oc_apply(path, manifest)` helper writes the manifest YAML to disk before applying it. + If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys), + it MUST NOT be written to any path under `env.ARTIFACT_DIR`. + +2. **Use `oc_apply_secret()` or `oc("apply", "-f", "-", input_text=...)` for secrets.** + Apply secrets via stdin or a dedicated helper that does not persist the payload. + Never use the standard `oc_apply()` for Secret-kind manifests. + +3. **Never log secret values.** + Do not print, log, or include in error messages: passwords, tokens, bearer credentials, + TLS keys, pull-secret `.dockerconfigjson`, or any `data:`/`stringData:` field from a Secret. + +4. **Check the `kind:` field.** + Before calling `oc_apply(artifact_path, manifest)`, verify that `manifest.get("kind") != "Secret"`. + If it is a Secret, use a non-persisting apply path. + +5. **Artifact directories are public.** + Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. + Only write non-sensitive operational data (CRs, logs, pod descriptions, events). + +### Safe Patterns + +```python +# GOOD: Apply a Secret without persisting to disk +def apply_secret(manifest: dict) -> None: + """Apply a Secret manifest via stdin — nothing written to disk.""" + import yaml + from projects.core.dsl.utils.k8s import oc + oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False)) + +# GOOD: Guard before using oc_apply +def safe_oc_apply(artifact_path, manifest): + if manifest.get("kind") == "Secret": + raise ValueError( + f"Refusing to write Secret '{manifest['metadata']['name']}' to artifacts. " + "Use apply_secret() instead." + ) + oc_apply(artifact_path, manifest) +``` + +### Forbidden Patterns + +```python +# BAD: Writing a pull-secret to artifacts +src_dir = env.ARTIFACT_DIR / "src" +oc_apply(src_dir / "image-pull-secret.yaml", secret_manifest) # LEAKS CREDENTIALS + +# BAD: Logging secret content +logger.info(f"Applying secret: {manifest}") # LEAKS if manifest has sensitive data + +# BAD: Including tokens in error messages +raise RuntimeError(f"Failed to auth with token {token}") # LEAKS TOKEN +``` From dca57ba777803b4dcc4afd7bd67c210e26d2f33f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 10:09:06 +0200 Subject: [PATCH 0147/1072] Apply suggestion from @kpouget --- AGENTS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 6da05ab1d..35943c693 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,10 @@ FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on Ope It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging. Key paths: -- `projects/` — individual project toolboxes (install scripts, test harnesses) +- `projects/` — individual project test harnesses +- `projects/NAME/orchestration` — upper layer of the test harness (config, prepare, test, cleanup) +- `projects/NAME/toolbox` — lower layer of the test harness (toolbox of scripts altering the state of the cluster, using the DSL language) +- `projects/NAME/test` — unit testing of the test harness - `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers) - `fournos/gitops/` — Tekton pipelines and GitOps base workflows - `docs/` — project documentation From b328d550a650d3630f519a5d5afdfb2c4aa6f3b8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 10:14:18 +0200 Subject: [PATCH 0148/1072] Apply suggestion from @kpouget --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 35943c693..4ba19d7a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,12 @@ Key paths: Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. Only write non-sensitive operational data (CRs, logs, pod descriptions, events). +6. **Use the vault module.** + Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret. + +7. **Never pass secrets via environment variables** + For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care. + ### Safe Patterns ```python From cee42b3bb73a1f04f3472d80f7c2f96beffa9420 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 17:09:36 -0400 Subject: [PATCH 0149/1072] Remove slack-webhook-url, use topsail bot exclusively for notifications Drop webhook_vault parameter from notification functions and callers, remove slack-webhook-url from psap-forge-notifications vault definition. Update default workload rates and durations. Co-authored-by: Cursor --- projects/rhaiis/orchestration/ci.py | 2 +- .../rhaiis/orchestration/config.d/workloads.yaml | 12 ++++++------ projects/rhaiis/orchestration/test_phase.py | 3 +-- projects/rhaiis/postprocess/regression.py | 15 ++++----------- vaults/psap-forge-notifications.yaml | 5 ----- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 25ea42404..763bca95c 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -74,7 +74,7 @@ def _check_pipeline_failure_and_notify() -> None: accelerator=accelerator_key, job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", + notification_vault="psap-forge-notifications", version=_cfg.project.get_config("tests.rhaiis.version", ""), cluster=cluster_tag, ) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index c413b505e..1cee1e4c6 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,13 +5,13 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [50] - max_seconds: 150 + rates: [1,50,100,200,300] + max_seconds: 450 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" - rates: [1] - max_seconds: 45 + rates: [1,50,100,200,300] + max_seconds: 450 profile3: data: "prompt_tokens=2048,output_tokens=128" @@ -20,7 +20,7 @@ profile3: profile4: data: "prompt_tokens=8000,output_tokens=1000" - rates: [1] - max_seconds: 45 + rates: [1,25,50,75,100] + max_seconds: 450 samples: 50 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 709a03f91..b47d9a176 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -41,7 +41,7 @@ def _send_pipeline_failure_alert( accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", + notification_vault="psap-forge-notifications", tp=str(vllm_args.get("tensor-parallel-size", "")), dp=str(vllm_args.get("data-parallel-size", "")), version=_cfg.project.get_config("tests.rhaiis.version", ""), @@ -702,7 +702,6 @@ def _run_regression_check( accelerator=accelerator, job_id=run_uuid, slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - webhook_vault="psap-forge-notifications", notification_vault="psap-forge-notifications", report_url=report_url, tp=str(_vllm.get("tensor-parallel-size", "")), diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 2c70abc5b..1b524f7fa 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -107,7 +107,6 @@ def send_regression_notification( accelerator: str = "", job_id: str = "", slack_user: str = "", - webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, report_url: str = "", @@ -116,17 +115,13 @@ def send_regression_notification( ) -> bool: """Send regression/improvement Slack notification from analysis results. - Uses a webhook URL (from *webhook_vault*) when available, falling - back to the Bot-Token path (from *notification_vault*). - Args: analysis_result: Output from run_regression_analysis() model: Model name for display accelerator: Accelerator name for display job_id: Job identifier slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials + notification_vault: Vault containing topsail-bot.slack-token dry_run: Log only, don't send report_url: Optional presigned URL to an agent analysis report tp: Tensor parallelism size for display @@ -231,7 +226,7 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) def send_failure_notification( @@ -241,7 +236,6 @@ def send_failure_notification( accelerator: str = "", job_id: str = "", slack_user: str = "", - webhook_vault: str | None = None, notification_vault: str | None = None, dry_run: bool = False, tp: str = "", @@ -258,8 +252,7 @@ def send_failure_notification( accelerator: Accelerator name for display job_id: FournosJob name slack_user: Slack user ID to @-mention - webhook_vault: Vault containing slack-webhook-url file - notification_vault: Fallback vault for Bot-Token Slack credentials + notification_vault: Vault containing topsail-bot.slack-token dry_run: Log only, don't send tp: Tensor parallelism size dp: Data parallelism size @@ -310,4 +303,4 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault or webhook_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 5646df832..12ae01cc8 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -12,8 +12,3 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot - slack-webhook-url: - description: | - Plain-text file containing a Slack Incoming Webhook URL - for regression/comparison alerts. - Example: https://hooks.slack.com/services/T.../B.../... From 57f08489ce85efa3f5e5b9c0489ca2b03c522e74 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 19:24:10 -0400 Subject: [PATCH 0150/1072] Refactor: extract analysis, notifications; configurable ISVC labels and S3 settings - Move analyze functions to orchestration/analysis.py - Move pipeline failure alert to orchestration/notifications.py - Make ISVC labels configurable via labels dict parameter - Move S3 bucket/vault/credentials to rhaiis.s3 config section - Remove hardcoded defaults from s3_dashboard.py functions - Drop redundant mode="w+b" from NamedTemporaryFile calls Co-authored-by: Cursor --- projects/rhaiis/orchestration/analysis.py | 278 ++++++++++++++ projects/rhaiis/orchestration/config.yaml | 11 +- .../rhaiis/orchestration/notifications.py | 49 +++ projects/rhaiis/orchestration/test_phase.py | 346 ++---------------- projects/rhaiis/postprocess/s3_dashboard.py | 24 +- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 1 + .../templates/inferenceservice.yaml.j2 | 7 +- 7 files changed, 379 insertions(+), 337 deletions(-) create mode 100644 projects/rhaiis/orchestration/analysis.py create mode 100644 projects/rhaiis/orchestration/notifications.py diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py new file mode 100644 index 000000000..5f22c36ac --- /dev/null +++ b/projects/rhaiis/orchestration/analysis.py @@ -0,0 +1,278 @@ +"""RHAIIS regression analysis and agent analysis helpers.""" + +from __future__ import annotations + +import logging +import os +import tempfile +from pathlib import Path + +from projects.core.library import env +from projects.rhaiis.orchestration import runtime_config + +logger = logging.getLogger(__name__) + + +def run_standalone_analysis( + model_cfg: dict, + accelerator_key: str, + vllm_args: dict, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, +) -> None: + """Run regression check + agent analysis using existing S3 data (no new benchmark).""" + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if not agent_cfg.get("enabled", False): + logger.info("Standalone analysis skipped: agent_analysis not enabled") + return + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not version or not compare_version: + logger.info("Standalone analysis skipped: version or compare_version not configured") + return + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping standalone analysis") + return + + accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + + consolidated_path = None + current_csv_path = None + try: + import pandas as pd + + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + + df = pd.read_csv(consolidated_path, on_bad_lines="warn") + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].str.strip() + + model_id = model_cfg.get("hf_model_id", "") + tp = str(vllm_args.get("tensor-parallel-size", 1)) + + current_rows = df[ + (df["version"] == version) + & (df["model"] == model_id) + & (df["accelerator"] == accelerator) + & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) + ] + + if current_rows.empty: + logger.warning( + "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", + version, model_id, accelerator, tp, + ) + return + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + current_csv_path = tmp.name + current_rows.to_csv(tmp, index=False) + + logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) + run_regression_check( + current_csv_path, compare_version, version, model_cfg, accelerator, + run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + if current_csv_path and os.path.exists(current_csv_path): + os.unlink(current_csv_path) + + +def run_regression_check( + csv_path, + compare_version: str, + current_version: str, + model_cfg: dict, + accelerator: str, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, + vllm_args: dict | None = None, +) -> None: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.caliper.engine.kpi.analyze import run_regression_analysis + from projects.core.library import config + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping regression analysis") + return + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + except Exception as e: + logger.warning("Could not download consolidated CSV for regression check: %s", e) + return + + try: + output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" + analysis = run_regression_analysis( + current_csv_path=Path(str(csv_path)), + consolidated_csv_path=Path(consolidated_path), + compare_version=compare_version, + current_version=current_version, + output_file=output_file, + profile_map=PROFILE_MAP, + metrics=METRICS, + restrict_profiles=restrict_profiles, + ) + + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + report_url = "" + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if agent_cfg.get("enabled", False): + report_url = run_agent_analysis( + analysis, model_cfg, accelerator, + current_version, compare_version, run_uuid, + severity_threshold=agent_cfg.get("severity_threshold", 10), + ) + + from projects.rhaiis.postprocess.regression import send_regression_notification + + _vllm = vllm_args or {} + send_regression_notification( + analysis, + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=run_uuid, + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + report_url=report_url, + tp=str(_vllm.get("tensor-parallel-size", "")), + dp=str(_vllm.get("data-parallel-size", "")), + ) + except Exception: + logger.warning("Regression analysis failed; continuing", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def run_agent_analysis( + analysis: dict, + model_cfg: dict, + accelerator: str, + current_version: str, + compare_version: str, + run_uuid: str, + *, + severity_threshold: int = 10, +) -> str: + """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" + from projects.core.library import config + from projects.rhaiis.postprocess.agent import ( + AGENT_SEVERITY_THRESHOLD, + build_pr_followup_prompt, + check_agent_connectivity, + markdown_to_html, + request_agent_analysis, + send_followup, + ) + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + agent_url = agent_cfg.get("url", "") + if not agent_url: + logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") + return "" + + threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD + severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] + if not severe: + logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) + return "" + + ok, detail = check_agent_connectivity(agent_url) + if not ok: + logger.warning("Agent not reachable, skipping analysis: %s", detail) + return "" + + tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + model = model_cfg.get("hf_model_id", "") + improvements = analysis.get("improvements", []) + + agent_response = request_agent_analysis( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe, + job_id=run_uuid, + improvements=improvements if improvements else None, + agent_url=agent_url, + ) + if not agent_response: + return "" + + pr_prompt = build_pr_followup_prompt(current_version, compare_version) + pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) + if pr_analysis: + agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" + + html_content = markdown_to_html( + agent_response, run_uuid, model, current_version, compare_version, + ) + html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" + html_path.write_text(html_content, encoding="utf-8") + logger.info("Agent analysis saved to %s", html_path) + + try: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + credentials_path = get_aws_credentials(vault_name, credentials_file) + if credentials_path: + s3 = create_s3_client(credentials_path) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" + s3.upload_file( + str(html_path), s3_bucket, s3_key, + ExtraArgs={"ContentType": "text/html"}, + ) + report_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": s3_bucket, "Key": s3_key}, + ExpiresIn=2592000, + ) + logger.info("Agent analysis uploaded to S3, presigned URL generated") + return report_url + except Exception: + logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) + + return "" diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index c0477ff2f..f1d53b638 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -29,6 +29,15 @@ tests: compare_version: "" slack_user: "" +rhaiis: + s3: + bucket: psap-dashboard-data + vault: psap-forge-dashboard-s3 + credentials_file: aws.credentials + profiler: + enabled: false + s3_prefix: pytorch-profiles/rhaiis + caliper: postprocess: enabled: true @@ -45,9 +54,7 @@ caliper: output: kpis.json csv_dashboard: enabled: false - s3_bucket: psap-dashboard-data s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv - vault: psap-forge-dashboard-s3 analyze: enabled: false historical_kpis: historical_kpis diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py new file mode 100644 index 000000000..486c6c340 --- /dev/null +++ b/projects/rhaiis/orchestration/notifications.py @@ -0,0 +1,49 @@ +"""RHAIIS pipeline notification helpers.""" + +from __future__ import annotations + +import logging +import os +import traceback + +from projects.rhaiis.orchestration import runtime_config + +logger = logging.getLogger(__name__) + + +def send_pipeline_failure_alert( + exc: Exception, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert when the pipeline fails.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=traceback.format_exception_only(type(exc), exc)[-1].strip(), + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline failure alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b47d9a176..0cad2b784 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -2,7 +2,6 @@ import logging import os -import traceback import uuid as _uuid_mod from projects.core.library import env @@ -14,43 +13,6 @@ _K8S_NAME_MAX = 63 -def _send_pipeline_failure_alert( - exc: Exception, - *, - model_key: str, - workload_keys: list[str], -) -> None: - """Best-effort Slack alert when the pipeline fails.""" - try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - - send_failure_notification( - error=traceback.format_exception_only(type(exc), exc)[-1].strip(), - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) - except Exception: - logger.warning("Failed to send pipeline failure alert", exc_info=True) - def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" @@ -92,7 +54,8 @@ def do_test( deployment_name=deployment_name, ) except Exception as exc: - _send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) + from projects.rhaiis.orchestration.notifications import send_pipeline_failure_alert + send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) raise @@ -168,7 +131,8 @@ def _run_test( if not run_benchmark and not profiler_enabled: logger.info("run_benchmark=false and profiler=false, running standalone analysis only") try: - _run_standalone_analysis( + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis( model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) @@ -180,6 +144,10 @@ def _run_test( wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) try: + isvc_labels = {"opendatahub.io/dashboard": "true"} + if profiler_enabled: + isvc_labels["vllm-profiler/enabled"] = "true" + logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) deploy_kserve_isvc( deployment_name=deployment_name, @@ -196,6 +164,7 @@ def _run_test( storage_pvc=deploy_cfg.get("storage_pvc", ""), image_pull_secret=deploy_cfg.get("image_pull_secret", ""), service_account_name=deploy_cfg.get("service_account_name", ""), + labels=isvc_labels, ) logger.info("Waiting for InferenceService to be ready") @@ -336,7 +305,8 @@ def _run_workload_benchmark( if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") try: - _run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Standalone analysis failed; continuing", exc_info=True) else: @@ -526,11 +496,13 @@ def _generate_and_sync_dashboard_csv( from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 + s3_cfg = config.project.get_config("rhaiis.s3", {}) sync_result = sync_csv_to_s3( csv_path, - s3_bucket=csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data"), - s3_key=csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv"), - vault_name=csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3"), + s3_bucket=s3_cfg.get("bucket", ""), + s3_key=csv_dashboard_cfg.get("s3_key", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Dashboard CSV sync result: %s", sync_result) @@ -539,277 +511,10 @@ def _generate_and_sync_dashboard_csv( if not compare_version: return - _run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) + from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) -def _run_standalone_analysis( - model_cfg: dict, - accelerator_key: str, - vllm_args: dict, - *, - run_uuid: str = "", - restrict_profiles: list[str] | None = None, -) -> None: - """Run regression check + agent analysis using existing S3 data (no new benchmark).""" - import tempfile - from pathlib import Path - - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.core.library import config - - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - if not agent_cfg.get("enabled", False): - logger.info("Standalone analysis skipped: agent_analysis not enabled") - return - - version = config.project.get_config("tests.rhaiis.version", "") - compare_version = config.project.get_config("tests.rhaiis.compare_version", "") - if not version or not compare_version: - logger.info("Standalone analysis skipped: version or compare_version not configured") - return - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") - s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if not credentials_path: - logger.warning("AWS credentials not available, skipping standalone analysis") - return - - # CSV stores just the chip family (e.g. "H200"), not the full accelerator_key with cluster tag - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() - - consolidated_path = None - current_csv_path = None - try: - import pandas as pd - - s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: - consolidated_path = tmp.name - s3.download_file(s3_bucket, s3_key, consolidated_path) - - df = pd.read_csv(consolidated_path, on_bad_lines="warn") - for col in df.columns: - if df[col].dtype == object: - df[col] = df[col].str.strip() - - model_id = model_cfg.get("hf_model_id", "") - tp = str(vllm_args.get("tensor-parallel-size", 1)) - - current_rows = df[ - (df["version"] == version) - & (df["model"] == model_id) - & (df["accelerator"] == accelerator) - & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) - ] - - if current_rows.empty: - logger.warning( - "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", - version, model_id, accelerator, tp, - ) - return - - with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: - current_csv_path = tmp.name - current_rows.to_csv(tmp, index=False) - - logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) - _run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, - run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, - ) - except Exception: - logger.warning("Standalone analysis failed", exc_info=True) - finally: - if consolidated_path and os.path.exists(consolidated_path): - os.unlink(consolidated_path) - if current_csv_path and os.path.exists(current_csv_path): - os.unlink(current_csv_path) - - -def _run_regression_check( - csv_path, - compare_version: str, - current_version: str, - model_cfg: dict, - accelerator: str, - *, - run_uuid: str = "", - restrict_profiles: list[str] | None = None, - vllm_args: dict | None = None, -) -> None: - import tempfile - from pathlib import Path - - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.caliper.engine.kpi.analyze import run_regression_analysis - from projects.core.library import config - from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - s3_bucket = csv_dashboard_cfg.get("s3_bucket", "psap-dashboard-data") - s3_key = csv_dashboard_cfg.get("s3_key", "staging/rhaiis-dashboard/consolidated_dashboard.csv") - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if not credentials_path: - logger.warning("AWS credentials not available, skipping regression analysis") - return - - consolidated_path = None - try: - s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: - consolidated_path = tmp.name - s3.download_file(s3_bucket, s3_key, consolidated_path) - except Exception as e: - logger.warning("Could not download consolidated CSV for regression check: %s", e) - return - - try: - output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" - analysis = run_regression_analysis( - current_csv_path=Path(str(csv_path)), - consolidated_csv_path=Path(consolidated_path), - compare_version=compare_version, - current_version=current_version, - output_file=output_file, - profile_map=PROFILE_MAP, - metrics=METRICS, - restrict_profiles=restrict_profiles, - ) - - if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: - report_url = "" - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - if agent_cfg.get("enabled", False): - report_url = _run_agent_analysis( - analysis, model_cfg, accelerator, - current_version, compare_version, run_uuid, - severity_threshold=agent_cfg.get("severity_threshold", 10), - ) - - from projects.rhaiis.postprocess.regression import send_regression_notification - - _vllm = vllm_args or {} - send_regression_notification( - analysis, - model=model_cfg.get("hf_model_id", ""), - accelerator=accelerator, - job_id=run_uuid, - slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - report_url=report_url, - tp=str(_vllm.get("tensor-parallel-size", "")), - dp=str(_vllm.get("data-parallel-size", "")), - ) - except Exception: - logger.warning("Regression analysis failed; continuing", exc_info=True) - finally: - if consolidated_path and os.path.exists(consolidated_path): - os.unlink(consolidated_path) - - -def _run_agent_analysis( - analysis: dict, - model_cfg: dict, - accelerator: str, - current_version: str, - compare_version: str, - run_uuid: str, - *, - severity_threshold: int = 10, -) -> str: - """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" - from pathlib import Path - - from projects.core.library import config - from projects.rhaiis.postprocess.agent import ( - AGENT_SEVERITY_THRESHOLD, - build_pr_followup_prompt, - check_agent_connectivity, - markdown_to_html, - request_agent_analysis, - send_followup, - ) - - agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) - agent_url = agent_cfg.get("url", "") - if not agent_url: - logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") - return "" - - threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD - severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] - if not severe: - logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) - return "" - - ok, detail = check_agent_connectivity(agent_url) - if not ok: - logger.warning("Agent not reachable, skipping analysis: %s", detail) - return "" - - tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) - model = model_cfg.get("hf_model_id", "") - improvements = analysis.get("improvements", []) - - agent_response = request_agent_analysis( - model=model, - accelerator=accelerator, - current_version=current_version, - compare_version=compare_version, - tp=tp, - severe_regressions=severe, - job_id=run_uuid, - improvements=improvements if improvements else None, - agent_url=agent_url, - ) - if not agent_response: - return "" - - pr_prompt = build_pr_followup_prompt(current_version, compare_version) - pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) - if pr_analysis: - agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" - - html_content = markdown_to_html( - agent_response, run_uuid, model, current_version, compare_version, - ) - html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" - html_path.write_text(html_content, encoding="utf-8") - logger.info("Agent analysis saved to %s", html_path) - - try: - from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") - credentials_path = get_aws_credentials(vault_name, "aws.credentials") - if credentials_path: - s3 = create_s3_client(credentials_path) - s3_bucket = "psap-dashboard-data" - s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" - s3.upload_file( - str(html_path), s3_bucket, s3_key, - ExtraArgs={"ContentType": "text/html"}, - ) - report_url = s3.generate_presigned_url( - "get_object", - Params={"Bucket": s3_bucket, "Key": s3_key}, - ExpiresIn=2592000, - ) - logger.info("Agent analysis uploaded to S3, presigned URL generated") - return report_url - except Exception: - logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) - - return "" - def _upload_predictor_log(run_uuid: str) -> None: """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" @@ -824,13 +529,14 @@ def _upload_predictor_log(run_uuid: str) -> None: logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) return - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - vault_name = csv_dashboard_cfg.get("vault", "psap-forge-dashboard-s3") + s3_cfg = config.project.get_config("rhaiis.s3", {}) result = upload_predictor_log_to_s3( log_path, run_uuid=run_uuid, - vault_name=vault_name, + s3_bucket=s3_cfg.get("bucket", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Predictor log upload result: %s", result) @@ -1020,6 +726,7 @@ def _upload_profiler_traces( profile_labels = _infer_profile_labels_from_traces(trace_files) logger.info("Inferred profile labels from trace filenames: %s", profile_labels) + s3_cfg = config.project.get_config("rhaiis.s3", {}) result = upload_profiler_traces_to_s3( traces_dir, model_name=model_cfg.get("hf_model_id", ""), @@ -1027,9 +734,10 @@ def _upload_profiler_traces( tp_size=int(vllm_args.get("tensor-parallel-size", 1)), version=version, profile_labels=profile_labels, - s3_bucket=profiler_cfg.get("s3_bucket", "psap-dashboard-data"), - s3_prefix=profiler_cfg.get("s3_prefix", "pytorch-profiles/rhaiis"), - vault_name=profiler_cfg.get("vault", "psap-forge-dashboard-s3"), + s3_bucket=s3_cfg.get("bucket", ""), + s3_prefix=profiler_cfg.get("s3_prefix", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), dry_run=config.project.get_config("caliper.export.dry_run", False), ) logger.info("Profiler trace upload result: %s", result) diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index d8b3705a0..c648ce9e7 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -13,16 +13,13 @@ def sync_csv_to_s3( csv_path: Path, *, - s3_bucket: str = "psap-dashboard-data", - s3_key: str = "staging/rhaiis-dashboard/consolidated_dashboard.csv", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + s3_key: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: - """Download consolidated CSV from S3, append new rows, re-upload. - - Ported from model-furnace's ModelValidator._sync_csv_to_s3(). - """ + """Download consolidated CSV from S3, append new rows, re-upload.""" try: import pandas as pd except ImportError: @@ -42,7 +39,7 @@ def sync_csv_to_s3( try: s3 = create_s3_client(credentials_path) - with tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv", delete=False) as tmp: + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: consolidated_path = tmp.name try: @@ -87,15 +84,14 @@ def upload_profiler_traces_to_s3( tp_size: int, version: str, profile_labels: list[str], - s3_bucket: str = "psap-dashboard-data", - s3_prefix: str = "pytorch-profiles/rhaiis", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + s3_prefix: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: """Upload profiler traces to S3. - Ported from model-furnace's ModelValidator._upload_profiler_traces_to_s3(). Path: s3://{bucket}/{prefix}/{accel}/{model}/tp{N}/{version}/{profile_label}/ """ if not traces_dir.exists(): @@ -168,8 +164,8 @@ def upload_predictor_log_to_s3( log_path: Path, *, run_uuid: str, - s3_bucket: str = "psap-dashboard-data", - vault_name: str = "psap-forge-dashboard-s3", + s3_bucket: str, + vault_name: str, credentials_file: str = "aws.credentials", dry_run: bool = False, ) -> dict[str, Any]: diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index c8cd621d9..e8908595a 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -26,6 +26,7 @@ def run( storage_pvc: str = "", image_pull_secret: str = "", service_account_name: str = "", + labels: dict | None = None, ): return execute_tasks(locals()) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index 3d04aa276..5ca599383 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -8,9 +8,12 @@ metadata: prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "8080" +{%- if args.labels %} labels: - opendatahub.io/dashboard: "true" - vllm-profiler/enabled: "true" +{%- for key, val in args.labels.items() %} + {{ key }}: "{{ val }}" +{%- endfor %} +{%- endif %} name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: From 069859a7710daae539a4b99c6cc91a7f9e29d296 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 21 Jul 2026 21:39:18 -0400 Subject: [PATCH 0151/1072] Fix config collision: move rhaiis.s3 to config.d/rhaiis.yaml Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 5 +++++ projects/rhaiis/orchestration/config.yaml | 9 --------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 12da18a24..90faeab5b 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,6 +36,11 @@ accelerator_env_vars: amd: VLLM_ROCM_USE_AITER: "1" +s3: + bucket: psap-dashboard-data + vault: psap-forge-dashboard-s3 + credentials_file: aws.credentials + warmup: rate: 200 max_seconds: 60 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index f1d53b638..9c6bf6f99 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -29,15 +29,6 @@ tests: compare_version: "" slack_user: "" -rhaiis: - s3: - bucket: psap-dashboard-data - vault: psap-forge-dashboard-s3 - credentials_file: aws.credentials - profiler: - enabled: false - s3_prefix: pytorch-profiles/rhaiis - caliper: postprocess: enabled: true From c3a89abb6a2c37cfaae8f38df4547513769b3784 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 11:51:51 +0300 Subject: [PATCH 0152/1072] refactor: config-driven nightly pipeline with reusable MLflow verifier Address PR #133 review feedback: - Remove skip_phases mechanism, let vault system handle nightly normally - Move MLflow verifier to caliper.orchestration for cross-project reuse - Use @requires config annotation for handler and verifier - Make receiver/verifier selection config-driven (dotted class paths) - Move GHCR repo/image from hardcoded constants to nightly.sources config - Replace NIGHTLY_SOURCE env var with configOverrides in CronJob - Add per-source mlflow_experiment for routing results to correct experiment - Add /pipeline nightly PR directive for source selection - Use vault.get_vault_content_path() instead of manual secret path construction --- projects/agentic_tools/ci_base.py | 11 +-- .../orchestration}/mlflow_verifier.py | 69 +++++++++--------- projects/core/nightly/handler.py | 70 ++++++++++++------- .../mcp_gateway/nightly_resolvers/__init__.py | 64 +---------------- .../nightly_resolvers/cronjob-ghcr.yaml | 4 +- .../nightly_resolvers/ghcr_receiver.py | 37 ++++++---- .../mcp_gateway/orchestration/config.yaml | 12 +++- projects/mcp_gateway/orchestration/pr_args.py | 12 ++++ 8 files changed, 133 insertions(+), 146 deletions(-) rename projects/{mcp_gateway/nightly_resolvers => caliper/orchestration}/mlflow_verifier.py (59%) diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index 3c078ab70..22247beb3 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -127,19 +127,14 @@ def list_vaults(self) -> list[str]: if isinstance(vault_config, list): return vault_config all_vaults: list[str] = [] - for category, vaults in vault_config.items(): - if category == "skip_phases": - continue + for _category, vaults in vault_config.items(): if isinstance(vaults, list): all_vaults.extend(vaults) return list(dict.fromkeys(all_vaults)) def should_init_vaults(self, phase: str) -> bool: - """Return False to skip vault init for resolve and phases listed in vaults.skip_phases.""" - if phase == "resolve-fournos-config": - return False - skip_phases = config.project.get_config("vaults.skip_phases", [], print=False) - return phase not in skip_phases + """Return False to skip vault init for the resolve phase.""" + return phase != "resolve-fournos-config" def register_extra_commands(self, group: click.Group) -> None: """Hook to register additional click commands on the group. diff --git a/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py similarity index 59% rename from projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py rename to projects/caliper/orchestration/mlflow_verifier.py index 2e9628a77..6a4b50bf8 100644 --- a/projects/mcp_gateway/nightly_resolvers/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -1,9 +1,15 @@ -"""MLflow nightly verifier for mcp_gateway. +"""Reusable MLflow nightly verifier. Connects to the MLflow tracking server, queries runs in the project's experiment, and extracts the version from the most recent run name. -Uses the same secret loading mechanism as caliper's mlflow export. +Projects reuse this by subclassing in their nightly_resolvers/ package +and setting NAME = "mlflow". The version extraction pattern is read from +``nightly.mlflow_verifier.version_pattern`` (regex with one capture group) +and an optional ``nightly.mlflow_verifier.strip_prefix``. + +All MLflow connection settings come from the caliper export config that +every FORGE project already has. """ from __future__ import annotations @@ -12,45 +18,47 @@ import os import re import time -from pathlib import Path from projects.caliper.engine.file_export.mlflow_secrets import ( load_mlflow_secrets_yaml, mlflow_connection_env, validate_mlflow_secrets, ) +from projects.core.library import vault +from projects.core.library.config import requires from projects.core.nightly.base_verifier import NightlyVerifier logger = logging.getLogger(__name__) -VERSION_PATTERN = re.compile(r"-v(.+?)-\d{8}-\d{6}$") - MAX_ATTEMPTS = 3 RETRY_DELAY_SECONDS = 15 -class MLflowVerifier(NightlyVerifier): +class CaliperMLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. - Reads its configuration from the project's config.yaml under: + Config requirements (via @requires on __init__): caliper.export.backend.mlflow.config.experiment caliper.export.backend.mlflow.config.workspace - caliper.export.backend.mlflow.secrets.vault.name - caliper.export.backend.mlflow.secrets.vault.mlflow_secret + caliper.export.backend.mlflow.secrets.vault (dict with 'name' and 'mlflow_secret') + nightly.mlflow_verifier.version_pattern (regex with one capture group) + nightly.mlflow_verifier.strip_prefix (prefix to remove from captured version) """ - NAME = "mlflow" - - def __init__(self): - from projects.core.library import config - - self.experiment = config.project.get_config( - "caliper.export.backend.mlflow.config.experiment" - ) - self.workspace = config.project.get_config("caliper.export.backend.mlflow.config.workspace") - vault_cfg = config.project.get_config("caliper.export.backend.mlflow.secrets.vault") - self.secret_name = vault_cfg["name"] - self.secret_file = vault_cfg["mlflow_secret"] + @requires( + experiment="caliper.export.backend.mlflow.config.experiment", + workspace="caliper.export.backend.mlflow.config.workspace", + vault_cfg="caliper.export.backend.mlflow.secrets.vault", + version_pattern="nightly.mlflow_verifier.version_pattern", + strip_prefix="nightly.mlflow_verifier.strip_prefix", + ) + def __init__(self, _cfg): + self.experiment = _cfg.experiment + self.workspace = _cfg.workspace + self.secret_name = _cfg.vault_cfg["name"] + self.secret_file = _cfg.vault_cfg["mlflow_secret"] + self._version_re = re.compile(_cfg.version_pattern) + self._strip_prefix = _cfg.strip_prefix or "" def get_last_tested_version(self) -> str: connection = self._load_connection() @@ -76,11 +84,10 @@ def get_last_tested_version(self) -> str: raise RuntimeError(f"MLflow verifier failed after {MAX_ATTEMPTS} attempts") from last_error def _load_connection(self) -> dict: - secrets_dir = Path(os.environ.get("FOURNOS_SECRETS", "/var/run/secrets/fournos")) - secret_path = secrets_dir / self.secret_name / self.secret_file + secret_path = vault.get_vault_content_path(self.secret_name, self.secret_file) - if not secret_path.exists(): - logger.warning("MLflow secret not found at expected path, assuming no previous run") + if secret_path is None or not secret_path.exists(): + logger.warning("MLflow secret not found in vault, assuming no previous run") return {} data = load_mlflow_secrets_yaml(secret_path) @@ -117,20 +124,14 @@ def _query_latest_run(self) -> str: return self._extract_version(run_name) def _extract_version(self, run_name: str) -> str: - """Extract the version from a run name like: - forge-mcp-gateway-s150-u500-vsha--20260714-224646 - - Returns the raw SHA (without the 'sha-' prefix) to match - what the receiver returns. - """ - match = VERSION_PATTERN.search(run_name) + match = self._version_re.search(run_name) if not match: logger.warning("Could not extract version from run name: %s", run_name) return "" version = match.group(1) - if version.startswith("sha-"): - return version[4:] + if self._strip_prefix and version.startswith(self._strip_prefix): + return version[len(self._strip_prefix):] return version diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 2ea3e3286..02a536efe 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -15,14 +15,28 @@ from projects.core.library import ci as ci_lib from projects.core.library import config, env +from projects.core.library.config import requires logger = logging.getLogger(__name__) -def _receive_image(project: str) -> str: - """Resolve the latest available version from the project's receiver.""" - module = importlib.import_module(f"projects.{project}.nightly_resolvers") - receiver = module.get_receiver() +def _get_source_config() -> dict: + """Read the source-specific config from nightly.sources..""" + source = config.project.get_config("nightly.source") + return config.project.get_config(f"nightly.sources.{source}") + + +def _resolve_class(dotted_path: str): + """Import a class from a dotted path like 'projects.foo.bar.MyClass'.""" + module_path, class_name = dotted_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + return getattr(mod, class_name) + + +def _receive_image(source_cfg: dict) -> str: + """Resolve the latest available version from the source's receiver.""" + receiver_cls = _resolve_class(source_cfg["receiver"]) + receiver = receiver_cls() logger.info("Using receiver: %s", type(receiver).__name__) version = receiver.get_latest_version() @@ -35,10 +49,10 @@ def _receive_image(project: str) -> str: return version -def _confirm_nightly(project: str, image_version: str) -> None: +def _confirm_nightly(project: str, image_version: str, source_cfg: dict) -> None: """Compare resolved version against last tested, create FournosJob if new.""" - module = importlib.import_module(f"projects.{project}.nightly_resolvers") - verifier = module.get_verifier() + verifier_cls = _resolve_class(config.project.get_config("nightly.verifier")) + verifier = verifier_cls() logger.info("Using verifier: %s", type(verifier).__name__) last_version = verifier.get_last_tested_version() @@ -53,43 +67,50 @@ def _confirm_nightly(project: str, image_version: str) -> None: return logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") - _create_fournos_job(project, image_version) + _create_fournos_job(project, image_version, source_cfg) result_msg = f"FournosJob created for {project} version {image_version}" ci_lib.add_notification_file("nightly-triggered", result_msg) (env.ARTIFACT_DIR / "result.txt").write_text(result_msg) -def _create_fournos_job(project: str, version: str) -> None: +@requires( + cluster="nightly.cluster", + version_key="nightly.version_key", + pipeline="nightly.pipeline", + preset="nightly.preset", + owner="nightly.owner", +) +def _create_fournos_job(project: str, version: str, source_cfg: dict, _cfg) -> None: """Create a FournosJob for the full test run using the fournos_launcher toolbox.""" from projects.fournos_launcher.toolbox.submit_and_wait.main import run as submit_and_wait namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") - cluster = config.project.get_config("nightly.cluster") - version_key = config.project.get_config("nightly.version_key") - pipeline = config.project.get_config("nightly.pipeline") - preset = config.project.get_config("nightly.preset", "") - owner = config.project.get_config("nightly.owner", "nightly-pipeline") - args_list = [preset] if preset else [] + args_list = [_cfg.preset] if _cfg.preset else [] + + overrides = {_cfg.version_key: version} + if mlflow_experiment := source_cfg.get("mlflow_experiment"): + overrides["caliper.export.backend.mlflow.config.experiment"] = mlflow_experiment + logger.info("MLflow experiment override: %s", mlflow_experiment) logger.info( "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", project, version, - cluster, - pipeline, + _cfg.cluster, + _cfg.pipeline, ) submit_and_wait( project, - cluster_name=cluster, + cluster_name=_cfg.cluster, args=args_list, - variables_overrides={version_key: version}, + variables_overrides=overrides, namespace=namespace, - owner=owner, + owner=_cfg.owner, display_name=f"nightly {project} {version}", - pipeline_name=pipeline, + pipeline_name=_cfg.pipeline, exclusive=True, wait=False, ) @@ -100,17 +121,18 @@ def run(): logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") project = config.project.get_config("project.name") - logger.info("Project: %s", project) + source_cfg = _get_source_config() + logger.info("Project: %s, source config: %s", project, source_cfg) try: - image_version = _receive_image(project) + image_version = _receive_image(source_cfg) except Exception as e: logger.error("receive-image failed: %s", e) ci_lib.add_notification_file("nightly-receive-image-failed", f"FATAL: {e}") raise try: - _confirm_nightly(project, image_version) + _confirm_nightly(project, image_version, source_cfg) except Exception as e: logger.error("confirm-nightly failed: %s", e) ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") diff --git a/projects/mcp_gateway/nightly_resolvers/__init__.py b/projects/mcp_gateway/nightly_resolvers/__init__.py index 27c4f1e56..73ebd8af5 100644 --- a/projects/mcp_gateway/nightly_resolvers/__init__.py +++ b/projects/mcp_gateway/nightly_resolvers/__init__.py @@ -1,65 +1,5 @@ """MCP Gateway nightly resolvers. -Auto-discovers ImageReceiver and NightlyVerifier subclasses in this package. -To add a new receiver or verifier, just create a new file with a class that: - - Inherits from ImageReceiver or NightlyVerifier - - Defines a NAME class attribute (used for selection) - -No modifications to this file are needed. +Contains ImageReceiver implementations for different image sources. +Receiver selection is config-driven via nightly.sources..receiver. """ - -from __future__ import annotations - -import importlib -import os -import pkgutil - -from projects.core.nightly.base_receiver import ImageReceiver -from projects.core.nightly.base_verifier import NightlyVerifier - -_receivers: dict[str, type[ImageReceiver]] = {} -_verifiers: dict[str, type[NightlyVerifier]] = {} - - -def _discover(): - """Import all modules in this package and register subclasses.""" - if _receivers or _verifiers: - return - - for _finder, module_name, _ in pkgutil.iter_modules(__path__): - module = importlib.import_module(f"{__name__}.{module_name}") - for attr in vars(module).values(): - if not isinstance(attr, type): - continue - if issubclass(attr, ImageReceiver) and attr is not ImageReceiver: - name = getattr(attr, "NAME", None) - if name: - _receivers[name] = attr - if issubclass(attr, NightlyVerifier) and attr is not NightlyVerifier: - name = getattr(attr, "NAME", None) - if name: - _verifiers[name] = attr - - -def get_receiver() -> ImageReceiver: - """Return the image receiver based on NIGHTLY_SOURCE env var.""" - _discover() - source = os.environ.get("NIGHTLY_SOURCE", "ghcr") - - cls = _receivers.get(source) - if cls is None: - raise ValueError( - f"Unknown source '{source}' for mcp_gateway. Available: {list(_receivers.keys())}" - ) - return cls() - - -def get_verifier() -> NightlyVerifier: - """Return the nightly verifier (first discovered NightlyVerifier subclass).""" - _discover() - - if not _verifiers: - raise RuntimeError("No NightlyVerifier subclass found in resolvers package") - - cls = next(iter(_verifiers.values())) - return cls() diff --git a/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml index bf053c29a..1eb501472 100644 --- a/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml +++ b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml @@ -40,8 +40,8 @@ spec: clusterless: true exclusive: false pipeline: "nightly" - env: - NIGHTLY_SOURCE: "ghcr" + configOverrides: + nightly.source: "ghcr" executionEngine: forge: project: "mcp_gateway" diff --git a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index 949de372f..eac5a8026 100644 --- a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -1,10 +1,14 @@ """GHCR image receiver for mcp_gateway. -Resolves the latest mcp-gateway commit SHA that has a published image -on ghcr.io. Checks recent commits from the repo and finds the first +Resolves the latest commit SHA that has a published image on ghcr.io. +Checks recent commits from the configured repo and finds the first one with a corresponding container image tag (sha-). No authentication needed for public packages (anonymous token). + +Configuration (in config.yaml under nightly.sources.ghcr): + repo: GitHub repo in "owner/name" format (e.g. "Kuadrant/mcp-gateway") + image: GHCR image path without registry (e.g. "kuadrant/mcp-gateway") """ from __future__ import annotations @@ -19,21 +23,26 @@ logger = logging.getLogger(__name__) -REPO = "Kuadrant/mcp-gateway" -IMAGE = "kuadrant/mcp-gateway" -COMMITS_URL = f"https://api.github.com/repos/{REPO}/commits?per_page=20" -TOKEN_URL = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{IMAGE}:pull" -MANIFEST_URL = f"https://ghcr.io/v2/{IMAGE}/manifests/" - MAX_ATTEMPTS = 3 RETRY_DELAY_SECONDS = 10 class GHCRReceiver(ImageReceiver): - """Find the latest mcp-gateway commit with a published ghcr.io image.""" + """Find the latest commit with a published ghcr.io image.""" NAME = "ghcr" + def __init__(self): + from projects.core.library import config + + source_cfg = config.project.get_config("nightly.sources.ghcr") + self.repo = source_cfg["repo"] + self.image = source_cfg["image"] + + self._commits_url = f"https://api.github.com/repos/{self.repo}/commits?per_page=20" + self._token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + self._manifest_url = f"https://ghcr.io/v2/{self.image}/manifests/" + def get_latest_version(self) -> str: last_error = None for attempt in range(1, MAX_ATTEMPTS + 1): @@ -66,12 +75,12 @@ def _fetch_version(self) -> str: ) def _get_ghcr_token(self) -> str: - with urllib.request.urlopen(TOKEN_URL, timeout=30) as resp: + with urllib.request.urlopen(self._token_url, timeout=30) as resp: data = json.loads(resp.read()) return data["token"] def _get_recent_commits(self) -> list[str]: - req = urllib.request.Request(COMMITS_URL) + req = urllib.request.Request(self._commits_url) with urllib.request.urlopen(req, timeout=30) as resp: commits = json.loads(resp.read()) return [c["sha"] for c in commits] @@ -79,7 +88,7 @@ def _get_recent_commits(self) -> list[str]: def _image_exists(self, sha: str, token: str) -> bool: tag = f"sha-{sha}" req = urllib.request.Request( - MANIFEST_URL + tag, + self._manifest_url + tag, headers={ "Accept": "application/vnd.oci.image.index.v1+json, " "application/vnd.docker.distribution.manifest.v2+json", @@ -87,8 +96,8 @@ def _image_exists(self, sha: str, token: str) -> bool: }, ) try: - urllib.request.urlopen(req, timeout=30) - return True + with urllib.request.urlopen(req, timeout=30): + return True except urllib.error.HTTPError as e: if e.code == 404: return False diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 8abbba249..7b33d74c9 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,4 +1,14 @@ nightly: + sources: + ghcr: + receiver: "projects.mcp_gateway.nightly_resolvers.ghcr_receiver.GHCRReceiver" + repo: "Kuadrant/mcp-gateway" + image: "kuadrant/mcp-gateway" + mlflow_experiment: "mcp-gw-cpt-ghcr" + verifier: "projects.caliper.orchestration.mlflow_verifier.CaliperMLflowVerifier" + mlflow_verifier: + version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" + strip_prefix: "sha-" cluster: "forge-smoke-testing" version_key: "infrastructure.mcp_gateway_version" pipeline: "forge-full" @@ -12,8 +22,6 @@ vaults: - psap-forge-mlflow-export prepare: [] test: [] - skip_phases: - - nightly notifications: slack: diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 82cffdb40..1b11aade6 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -9,6 +9,8 @@ /test fournos mcp_gateway smoke /test fournos mcp_gateway smoke collect_cluster_info /version 0.7.0 + /pipeline nightly ghcr + /pipeline nightly quay """ from __future__ import annotations @@ -77,4 +79,14 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st parsed_directives.append(line) logger.info("Parsed mcp_gateway version: %s", parts[0]) + # Parse /pipeline nightly — extract source from pipeline directive + if line.startswith("/pipeline nightly "): + parts = line.split() + if len(parts) == 3: + source = parts[2] + config_overrides["nightly.source"] = source + config_overrides["fournos.job.pipeline_name"] = "nightly" + parsed_directives.append(line) + logger.info("Parsed nightly source from pipeline directive: %s", source) + return config_overrides, parsed_directives From cf1ff57a877bb7e2830d86a323cde39e29d2de07 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 11:55:18 +0300 Subject: [PATCH 0153/1072] style: apply ruff formatting --- projects/caliper/orchestration/mlflow_verifier.py | 2 +- projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 6a4b50bf8..8d7d00adb 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -132,6 +132,6 @@ def _extract_version(self, run_name: str) -> str: version = match.group(1) if self._strip_prefix and version.startswith(self._strip_prefix): - return version[len(self._strip_prefix):] + return version[len(self._strip_prefix) :] return version diff --git a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py index eac5a8026..36853fc71 100644 --- a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -40,7 +40,9 @@ def __init__(self): self.image = source_cfg["image"] self._commits_url = f"https://api.github.com/repos/{self.repo}/commits?per_page=20" - self._token_url = f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + self._token_url = ( + f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + ) self._manifest_url = f"https://ghcr.io/v2/{self.image}/manifests/" def get_latest_version(self) -> str: From abfe64aaa2b8d2f833f2e71a1ef64a6af2e97513 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 22 Jul 2026 10:52:26 +0100 Subject: [PATCH 0154/1072] fix: Increase number of attempts Signed-off-by: Alberto Perdomo --- projects/fournos_launcher/toolbox/submit_and_wait/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 40e4917b6..9d205c410 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -211,7 +211,7 @@ def submit_fournos_job(args, ctx): return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" -@retry(attempts=360, delay=10, backoff=1.0) +@retry(attempts=3000, delay=10, backoff=1.0) @task def wait_for_job_completion(args, ctx): """Wait for FOURNOS job to complete""" From a7cea8ac07df78e2bbf18a2fe6da6476dc5ed004 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 22 Jul 2026 13:53:43 +0300 Subject: [PATCH 0155/1072] small fix to run in the smoke cluster --- projects/mcp_gateway/toolbox/install_platform/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index 6fdc75046..1bbd97fca 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -116,7 +116,7 @@ def install_service_mesh_instance(args, ctx): if "wait_for_ready" in step: spec = step["wait_for_ready"] ctx.mesh_ready_spec = spec - return (False, "Service Mesh instance applied, readiness checked in next task") + return "Service Mesh instance applied, readiness checked in next task" return f"Service Mesh instance applied from {kustomize_path}" From 6196820bd0d04bc400e3a9890f7f9e5c251d97bf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:16:40 +0200 Subject: [PATCH 0156/1072] [core] dsl: show the function duration in the completion banner --- projects/core/dsl/log.py | 22 +++++++++++++++++++++- projects/core/dsl/runtime.py | 13 +++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/projects/core/dsl/log.py b/projects/core/dsl/log.py index 0f3c2d18d..16f619975 100644 --- a/projects/core/dsl/log.py +++ b/projects/core/dsl/log.py @@ -104,7 +104,9 @@ def log_execution_banner(function_args: dict = None, log_file: str = None): logger.info("") -def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): +def log_completion_banner( + function_args: dict = None, status: str = "SUCCESS", start_time: float = None +): """Log the completion banner with function info and completion status""" frame = inspect.currentframe() caller_frame = frame.f_back.f_back @@ -118,6 +120,24 @@ def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): logger.info(f"| {rel_filename}") logger.info(f"| STATUS: {status}") logger.info(f"| COMMAND: {function_name}") + + # Add duration if start_time is provided + if start_time is not None: + end_time = time.time() + duration = end_time - start_time + minutes, seconds = divmod(duration, 60) + hours, minutes = divmod(minutes, 60) + + # Format duration nicely + if hours > 0: + duration_str = f"{int(hours)}h {int(minutes)}m {int(seconds)}s" + elif minutes > 0: + duration_str = f"{int(minutes)}m {int(seconds)}s" + else: + duration_str = f"{duration:.1f}s" + + logger.info(f"| DURATION: {duration_str}") + logger.info(f"| ARTIFACTS: {env.ARTIFACT_DIR}") logger.info("===============================================================================") logger.info("") diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index 024555f64..78ff145d6 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -174,7 +174,7 @@ def execute_tasks(function_args: dict = None): if not file_tasks: logger.error(f"No tasks found for file: {rel_filename}") - log_completion_banner(function_args, status="NO_TASKS") + log_completion_banner(function_args, status="NO_TASKS", start_time=start_time) raise RuntimeError(f"No tasks found for file: {rel_filename}") execution_error = None @@ -193,7 +193,7 @@ def execute_tasks(function_args: dict = None): logger.fatal("==> INTERRUPTED: Received KeyboardInterrupt (Ctrl+C)") logger.info("==> Exiting...") # Show completion banner with interrupted status - log_completion_banner(function_args, status="INTERRUPTED") + log_completion_banner(function_args, status="INTERRUPTED", start_time=start_time) raise except EarlyReturnException as e: @@ -213,7 +213,11 @@ def execute_tasks(function_args: dict = None): # Log error but continue to always tasks logger.info("") logger.fatal(f"==> {e.__class__.__name__}: {e}") - log_completion_banner(function_args, status=f"EXCEPTION ({e.__class__.__name__})") + log_completion_banner( + function_args, + status=f"EXCEPTION ({e.__class__.__name__})", + start_time=start_time, + ) # Save error to re-raise after always tasks execute execution_error = e @@ -278,6 +282,7 @@ def execute_tasks(function_args: dict = None): log_completion_banner( function_args, status=f"FAILED ({len(all_exceptions)} exception(s))", + start_time=start_time, ) if len(all_exceptions) == 1: raise all_exceptions[0] @@ -287,7 +292,7 @@ def execute_tasks(function_args: dict = None): ) # Log completion banner if execution was successful - log_completion_banner(function_args) + log_completion_banner(function_args, start_time=start_time) # for the time being, return the shared context. In the # future we'll return more info about what has been From 71c91ae450fd8ef6e6ef023e329f7f46ffac66fc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:38:19 +0200 Subject: [PATCH 0157/1072] [core] library: postprocess: skip in the postprocessing on keyboard interrupt --- projects/core/library/postprocess.py | 128 +++++++++++++++++---------- 1 file changed, 82 insertions(+), 46 deletions(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 893c9e7f9..fa2c8ddb0 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -143,17 +143,29 @@ def run_and_postprocess(test_func, *args, **kwargs): exc_msg: str | None = None ret: int | None = None original_exc: BaseException | None = None # Store the test exception + interrupted: bool = False try: ret = test_func(*args, **kwargs) return ret - except BaseException as e: + except (KeyboardInterrupt, BaseException) as e: + # Import here to avoid circular dependencies + from projects.core.library.run import SignalInterrupt + + # Check if this is an interrupt + if isinstance(e, KeyboardInterrupt | SignalInterrupt): + interrupted = True + logger.info("==> Test interrupted - skipping Caliper post-processing") + exc_msg = str(e) original_exc = e # Capture before the name is cleared raise finally: - # Determine test outcome based on exception/return code - if exc_msg is not None: + # Skip post-processing when interrupted to avoid blocking on finalizers + if interrupted: + logger.info("==> Skipping Caliper post-processing due to interrupt") + outcome = None + elif exc_msg is not None: outcome = TestPhaseOutcome("FAILED", exc_msg) elif ret == 0: outcome = TestPhaseOutcome("SUCCESS") @@ -164,56 +176,80 @@ def run_and_postprocess(test_func, *args, **kwargs): # Run postprocessing and check status for failures try: - status = run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) - - # Check if postprocessing failed - success = status.get("success", False) - if not success: - final_status = status.get("final_status", "unknown") - - # Check if failure is only due to warnings - if _is_warnings_only_failure(status): - if original_exc is not None: - # Test failed, postprocess has warnings: still fail due to test - logger.error( - "Test failed and postprocessing completed with warnings (final_status: %s)", - final_status, - ) - raise # Re-raise the original test exception - else: - # Test succeeded, postprocess has warnings only: treat as success - logger.warning( - "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", - final_status, - ) - return 0 + status = ( + run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) + if not interrupted + else dict(success=False, final_status="Test interrupted") + ) + + # Handle None status when postprocessing is disabled - treat as unsuccessful + if status is None or not status.get("success", False): + if status is None: + final_status = "postprocessing disabled" + else: + final_status = status.get("final_status", "unknown") + result = _handle_postprocess_failure(status, original_exc, final_status) + if result is None: + # Let original test exception propagate through outer flow + pass else: - # Actual postprocessing failures (not just warnings) - if original_exc is not None: - # Both test and postprocess failed: log both issues - logger.error( - "Both test and postprocessing failed (final_status: %s)", final_status - ) - raise # Re-raise the original test exception - else: - # Only postprocess failed: return failure code - logger.error( - "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", - final_status, - ) - return 1 + return result except Exception as postprocess_exc: logger.exception("Caliper postprocess after test failed with exception") if original_exc is not None: # Both test and postprocess failed: chain so both are visible in the traceback raise postprocess_exc from original_exc - else: - # Only postprocess failed: return failure code instead of raising - logger.error( - "Test succeeded but postprocessing failed with exception - returning exit code 1" - ) - return 1 + + # Only postprocess failed: return failure code instead of raising + logger.error( + "Test succeeded but postprocessing failed with exception - returning exit code 1" + ) + return 1 + + +def _handle_postprocess_failure( + status: dict, original_exc: BaseException | None, final_status: str +) -> int | None: + """Handle postprocessing failure logic. + + Args: + status: Postprocessing status dictionary + original_exc: Original test exception if any + final_status: Final status from postprocessing + + Returns: + Exit code to return, or None to re-raise original exception + """ + # Check if failure is only due to warnings + if _is_warnings_only_failure(status): + if original_exc is not None: + # Test failed, postprocess has warnings: still fail due to test + logger.error( + "Test failed and postprocessing completed with warnings (final_status: %s)", + final_status, + ) + return None # Signal to re-raise original exception + else: + # Test succeeded, postprocess has warnings only: treat as success + logger.warning( + "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", + final_status, + ) + return 0 + else: + # Actual postprocessing failures (not just warnings) + if original_exc is not None: + # Both test and postprocess failed: log both issues + logger.error("Both test and postprocessing failed (final_status: %s)", final_status) + return None # Signal to re-raise original exception + else: + # Only postprocess failed: return failure code + logger.error( + "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", + final_status, + ) + return 1 def _is_warnings_only_failure(status: dict) -> bool: From aaed126b9dd26c6625aba7cb80f0e5d1c9e9c93a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 11:52:39 +0200 Subject: [PATCH 0158/1072] [core] library: ci: better handle early exception before the env module is initialized --- projects/core/library/ci.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 475da31eb..894e5c7d4 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -108,11 +108,16 @@ def _display_error_summary(e: Exception) -> None: for line in summary_lines: logger.error(line) - if env.ARTIFACT_DIR is None: - logging.error("env.ARTIFACT_DIR not set, cannot generate the error summary file") - else: - # Write to FAILURES file - _write_error_summary_to_file(summary_lines) + # Attempt to write to FAILURES file if environment is initialized + try: + if env.ARTIFACT_DIR is not None: + _write_error_summary_to_file(summary_lines) + else: + logger.warning( + "Environment not fully initialized - error summary displayed above but not saved to file" + ) + except Exception as write_error: + logger.warning(f"Could not save error summary to file: {write_error}") logger.error("=" * 80) From 69062f65bb3cd34f40355d3108b0d8e8cad2ad75 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 13:58:29 +0200 Subject: [PATCH 0159/1072] [core] library: config: more more reliable if 'extends: None' --- projects/core/library/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index d0d0437cb..9dd5d51f2 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -210,7 +210,7 @@ def apply_preset(self, name): for key, value in values.items(): if key == "extends": - for extend_name in value: + for extend_name in value or []: self.apply_preset(extend_name) continue From 7affb8215557c1a949205132a6c6c1d09615b046 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:17:11 +0200 Subject: [PATCH 0160/1072] [core] dsl: utils: k8s: allow oc to save to file --- projects/core/dsl/utils/k8s.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 460ff0755..62a7f0985 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -23,6 +23,7 @@ def oc( timeout_seconds: float | None = 300, log_stdout: bool = True, log_stderr: bool = True, + stdout_dest: Path | None = None, ) -> shell.CommandResult: """Run an oc command. @@ -33,6 +34,7 @@ def oc( timeout_seconds: Command timeout (not supported) log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) + stdout_dest: Optional file path to write stdout to Returns: CommandResult with execution details @@ -45,6 +47,7 @@ def oc( shell=False, log_stdout=log_stdout, log_stderr=log_stderr, + stdout_dest=stdout_dest, ) From c9d19f4a5893684f6ab72705f80449286c4759df Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:22:51 +0200 Subject: [PATCH 0161/1072] [core] dsl: utils: k8s: use stdout_dest --- projects/core/dsl/utils/k8s.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 62a7f0985..82a283384 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -269,6 +269,7 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: logger.info("Capturing logs from %d pods in %s", len(pod_names), namespace) for pod_name in pod_names: + log_file = output_dir / f"{pod_name}.log" log_result = oc( "logs", pod_name, @@ -277,11 +278,10 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: "--all-containers=true", check=False, log_stdout=False, + stdout_dest=log_file, ) - log_file = output_dir / f"{pod_name}.log" - if log_result.returncode == 0 and log_result.stdout: - log_file.write_text(log_result.stdout, encoding="utf-8") - else: + # If command failed and no output was written, write error info + if log_result.returncode != 0 and (not log_file.exists() or log_file.stat().st_size == 0): log_file.write_text( f"(failed to collect logs: exit_code={log_result.returncode})\n" f"{log_result.stderr or ''}", From 5caaa6ca1312c4ac1943c56ee09dc4ef2f4f10be Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:23:46 +0200 Subject: [PATCH 0162/1072] [agentic_tools] mcp: toolbox: deploy_mock_servers/main: use stdout_dest --- .../agentic_tools/mcp/toolbox/deploy_mock_servers/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py index 6b65ecb11..19baedb65 100644 --- a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py +++ b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py @@ -292,6 +292,4 @@ def _generate_server_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) From 5a637299aad9ea1697d9785a53864f7acaa75ade Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:24:05 +0200 Subject: [PATCH 0163/1072] [guidellm] toolbox: run_guidellm_benchmark: use stdout_dest --- .../toolbox/run_guidellm_benchmark/main.py | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 91cbcc445..61f6cabc4 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -393,19 +393,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name ) # Capture job logs - result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job.logs", ) - if result.returncode == 0 and result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job.logs", result.stdout) # Capture additional debugging info - pods_result = oc( + oc( "get", "pods", "-n", @@ -415,11 +414,10 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_pods.yaml", ) - if pods_result.returncode == 0 and pods_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_pods.yaml", pods_result.stdout) - job_result = oc( + oc( "get", "job", benchmark_name, @@ -428,20 +426,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_detailed.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_detailed.yaml", job_result.stdout) - logs_result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_logs.txt", ) - if logs_result.returncode == 0 and logs_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_logs.txt", logs_result.stdout) def capture_get( @@ -460,9 +456,7 @@ def capture_get( if selector: args.extend(["-l", selector]) args.extend(["-o", output]) - result = oc(*args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + oc(*args, check=False, log_stdout=False, stdout_dest=destination) if __name__ == "__main__": From 9cbfac0208e97f77aa893cb502788da484198635 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:24:39 +0200 Subject: [PATCH 0164/1072] [agentic_tools] locust: toolbox: generate_prompts/main: use stdout_dest --- .../agentic_tools/locust/toolbox/generate_prompts/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py index c70ed336b..78504f9be 100644 --- a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py +++ b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py @@ -237,16 +237,15 @@ def capture_logs(args, ctx): artifacts_dir = env.ARTIFACT_DIR / "artifacts" / "prompts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc( + oc( "logs", f"job/{GENERATOR_JOB_NAME}", "-n", args.namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "generator.log", ) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "generator.log").write_text(result.stdout, encoding="utf-8") return "Logs captured" From 9790aa06663cc44f28a386bca39fd96bccda66c4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:25:10 +0200 Subject: [PATCH 0165/1072] [kserve] toolbox: prepare_hf_model_cache: use oc stdout_dest --- .../toolbox/prepare_hf_model_cache/main.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index 93c747a80..572ec43dc 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -11,7 +11,6 @@ slugify_identifier, truncate_k8s_name, write_json, - write_text, write_yaml, ) from projects.core.dsl.utils.k8s import ( @@ -365,7 +364,7 @@ def capture_download_artifacts(args, ctx): ) # Capture PVC YAML to file (not logs) - pvc_result = oc( + oc( "get", "persistentvolumeclaim", cache_spec["pvc_name"], @@ -376,12 +375,11 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "pvc.yaml", ) - if pvc_result.returncode == 0 and pvc_result.stdout: - write_text(artifact_dir / "pvc.yaml", pvc_result.stdout) # Capture Job YAML to file (not logs) - job_result = oc( + oc( "get", "job", cache_spec["download_job_name"], @@ -392,14 +390,13 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "job.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifact_dir / "job.yaml", job_result.stdout) # Capture pod logs and YAML to files (not console) for pod_name in job_pod_names(cache_spec["download_job_name"], cache_spec["namespace"]): # Get pod YAML to file - pod_result = oc( + oc( "get", "pod", pod_name, @@ -410,21 +407,19 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.yaml", ) - if pod_result.returncode == 0 and pod_result.stdout: - write_text(artifact_dir / f"{pod_name}.yaml", pod_result.stdout) # Get pod logs to file - log_result = oc( + oc( "logs", pod_name, "-n", cache_spec["namespace"], check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.log", ) - if log_result.returncode == 0 and log_result.stdout: - write_text(artifact_dir / f"{pod_name}.log", log_result.stdout) return f"Artifacts captured for {cache_spec['pvc_name']}" From 101b4b88b3f80939db5d6aa134b673d51f9afcc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:25:42 +0200 Subject: [PATCH 0166/1072] [llm_d] toolbox: capture_prepare_state: use stdout_dest --- .../llm_d/toolbox/capture_prepare_state/main.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/toolbox/capture_prepare_state/main.py b/projects/llm_d/toolbox/capture_prepare_state/main.py index 8d38c8783..f1e7a1248 100644 --- a/projects/llm_d/toolbox/capture_prepare_state/main.py +++ b/projects/llm_d/toolbox/capture_prepare_state/main.py @@ -6,7 +6,6 @@ from pathlib import Path from projects.core.dsl import entrypoint, execute_tasks, task -from projects.core.dsl.utils import write_text from projects.core.dsl.utils.k8s import oc logger = logging.getLogger(__name__) @@ -65,10 +64,10 @@ def capture_datasciencecluster_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured DataScienceCluster: {destination}" else: logger.warning("Failed to capture DataScienceCluster: %s", result.stderr) @@ -91,10 +90,10 @@ def capture_gateway_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway: {destination}" else: logger.warning("Failed to capture Gateway: %s", result.stderr) @@ -117,10 +116,10 @@ def capture_gateway_service_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway service: {destination}" else: logger.warning("Failed to capture Gateway service: %s", result.stderr) @@ -144,10 +143,10 @@ def capture_namespace_events(args, ctx): "--sort-by=.metadata.creationTimestamp", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured namespace events: {destination}" else: logger.warning("Failed to capture namespace events: %s", result.stderr) From d032774cfbc33d8df9eac057c4f1bdd00f92b13d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:26:07 +0200 Subject: [PATCH 0167/1072] [mcp_gateway] toolbox: use stdout_dest --- .../toolbox/apply_infrastructure/main.py | 4 +-- .../toolbox/cleanup_platform/main.py | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py index 353298467..062efe707 100644 --- a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py +++ b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py @@ -253,6 +253,4 @@ def _generate_infrastructure_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) diff --git a/projects/mcp_gateway/toolbox/cleanup_platform/main.py b/projects/mcp_gateway/toolbox/cleanup_platform/main.py index bbfe8ced2..f28860a0a 100644 --- a/projects/mcp_gateway/toolbox/cleanup_platform/main.py +++ b/projects/mcp_gateway/toolbox/cleanup_platform/main.py @@ -200,13 +200,26 @@ def capture_cleanup_state(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc("get", "csv", "-A", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-csvs.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "csv", + "-A", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-csvs.txt", + ) - result = oc("get", "namespaces", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-namespaces.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "namespaces", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-namespaces.txt", + ) return "Captured post-cleanup state" From 83cdf21ef40a1dbcbc5b1d134397c79e9b13d0af Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 15:26:31 +0200 Subject: [PATCH 0168/1072] [mcp_gateway] toolbox: install_platform: use stdout_dest --- projects/mcp_gateway/toolbox/install_platform/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index fcf5c5d5c..623b01fc3 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -524,9 +524,7 @@ def _get_mcp_host() -> str: def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) def _parse_version(version: str) -> tuple[int, ...]: From 885f6185656403b225d90ef9808a40506fa3062f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 17:40:00 +0200 Subject: [PATCH 0169/1072] [core] Add the config changelog --- projects/core/CHANGELOG-config.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 projects/core/CHANGELOG-config.md diff --git a/projects/core/CHANGELOG-config.md b/projects/core/CHANGELOG-config.md new file mode 100644 index 000000000..3fbf66f62 --- /dev/null +++ b/projects/core/CHANGELOG-config.md @@ -0,0 +1,17 @@ +# Configuration Framework Changelog + +## 2026-07-16 - Variables Override Helper + +### New Features +- **Variables Override Helper**: Added `write_variables_override()` function for programmatic configuration override generation + - **Behavior**: Creates variables_override.yaml files without requiring full project config initialization + - **Usage**: Supports both preset configuration (`project.args`) and additional variable overrides + - **Error Handling**: Validates `env.ARTIFACT_DIR` initialization with clear error messages + +### Files Modified +- `projects/core/library/config.py` - Added `write_variables_override()` helper function + +### Benefits +- **Early Configuration**: Enables configuration override before project initialization +- **Flexible Override Structure**: Supports both preset lists and arbitrary configuration variables +- **Robust Error Handling**: Fails fast with clear messages when environment not properly initialized From 80b3ac2faec268d2267b069a0e63e5cc2a314285 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 15 Jul 2026 17:24:34 +0200 Subject: [PATCH 0170/1072] [llm_d] start working on P/D testing --- projects/llm_d/docs/PD_TESTING.md | 381 ++++++++++++++++++ .../orchestration/config.d/deployments.yaml | 42 ++ .../manifests/deployments/pd-scheduler.yaml | 86 ++++ .../orchestration/presets.d/presets.yaml | 99 ++++- .../llm_d/orchestration/runtime_config.py | 29 ++ 5 files changed, 633 insertions(+), 4 deletions(-) create mode 100644 projects/llm_d/docs/PD_TESTING.md create mode 100644 projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml diff --git a/projects/llm_d/docs/PD_TESTING.md b/projects/llm_d/docs/PD_TESTING.md new file mode 100644 index 000000000..b62c752d5 --- /dev/null +++ b/projects/llm_d/docs/PD_TESTING.md @@ -0,0 +1,381 @@ +# P/D (Prefill/Decode) Testing in llm-d + +This document describes the P/D testing capabilities in the llm-d project, which allows testing various Prefill/Decode configurations with flexible pod counts, tensor parallelism, and scheduler configurations using profile name-based parameter extraction. + +## Overview + +P/D testing enables you to: +- Configure the number of prefill and decode pods via profile names +- Set different tensor parallelism for prefill vs decode pods +- Test various scheduler configurations +- Run matrix tests across multiple P/D configurations +- Use template-based profiles with automatic parameter extraction + +## P/D Configuration Architecture + +### Profile Name-Based Configuration + +P/D deployments use profile names to encode configuration parameters: + +**Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + +**Examples**: +- `pd-d.x2-p.tp1-d.tp4-p.x8` - 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-d.x1-p.tp2-d.tp8-p.x4` - 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +### Global P/D Configuration + +P/D-specific settings are centralized in `deployments.yaml`: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Template-Based Profiles + +Template profiles use `FROM_NAME` placeholders for automatic parameter extraction: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME # extracted from profile name + pd_config: + prefill_pods: FROM_NAME # extracted from p.x{N} pattern + decode_pods: FROM_NAME # extracted from d.x{N} pattern + scheduler_config: "high-throughput" +``` + +## P/D Profile Examples + +### Small Scale P/D +```yaml +# Profile: pd-d.x1-p.tp1-d.tp2-p.x2 +# 1 decode pod with 2 GPUs, 2 prefill pods with 1 GPU each +deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 +``` + +### Medium Scale P/D +```yaml +# Profile: pd-d.x2-p.tp2-d.tp4-p.x4 +# 2 decode pods with 4 GPUs each, 4 prefill pods with 2 GPUs each +deployment_profile: pd-d.x2-p.tp2-d.tp4-p.x4 +``` + +### Large Scale P/D +```yaml +# Profile: pd-d.x4-p.tp4-d.tp8-p.x8 +# 4 decode pods with 8 GPUs each, 8 prefill pods with 4 GPUs each +deployment_profile: pd-d.x4-p.tp4-d.tp8-p.x8 +``` + +## P/D Testing with Presets + +P/D testing integrates with the existing preset system. Use presets that include P/D profile configurations: + +### Custom P/D Testing +```bash +llm_d ci --preset your-pd-preset +``` + +Where your preset contains P/D profile configurations in `presets.yaml`: + +```yaml +your-pd-preset: + runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 + benchmark_key: short +``` + +## Custom P/D Testing via PR Comments + +You can override P/D configurations using PR comment directives: + +### Test Specific P/D Configurations +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +/var runtime.model_name: meta-llama/Llama-3.1-8B-Instruct +/var runtime.benchmark_key: short +``` + +### Test Different P/D Scales +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp4-p.x1, pd-d.x4-p.tp4-d.tp8-p.x8] +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.benchmark_key: [short, concurrent-1k-1k] +``` + +### Mix P/D and Traditional Profiles +``` +/var runtime.deployment_profile: [pd-d.x2-p.tp1-d.tp4-p.x8, approximate-prefix-cache, precise-prefix-cache] +/var runtime.model_name: Qwen/Qwen3-0.6B +``` + +## Parameter Extraction Rules + +The system extracts parameters from P/D profile names using these patterns: + +### Tensor Parallelism +- **Main container (decode)**: `d.tp{N}` or `tp{N}` → N GPUs per decode pod +- **Prefill container**: `p.tp{N}` → N GPUs per prefill pod + +### Pod Replicas +- **Main container (decode)**: `d.x{N}` or `x{N}` → N decode pods +- **Prefill container**: `p.x{N}` → N prefill pods + +### Examples +- `pd-d.x2-p.tp1-d.tp4-p.x8`: 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-tp8-x1-p.tp2-p.x4`: 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +## Runtime Configuration Functions + +New runtime configuration functions are available for P/D testing: + +```python +from projects.llm_d.orchestration import runtime_config + +# Get P/D configuration +pd_config = runtime_config.get_pd_config() +# Returns: {'prefill_pods': 2, 'decode_pods': 4, 'scheduler_config': 'high-throughput'} + +# Get individual P/D parameters +prefill_pods = runtime_config.get_prefill_pod_count() # Returns: 2 +decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 +scheduler_config = runtime_config.get_scheduler_config() # Returns: 'high-throughput' + +# Check if current deployment is P/D +is_pd = runtime_config.is_pd_deployment() # Returns: True for P/D profiles +``` + +## Test Output and Artifacts + +P/D tests generate the same artifacts as standard llm-d tests, with additional P/D-specific metadata: + +### Test Labels +```yaml +model_name: meta-llama/Llama-3.1-8B-Instruct +deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 +guidellm_loadshape: short +pd_prefill_pods: 8 # Extracted from p.x8 +pd_decode_pods: 2 # Extracted from d.x2 +pd_prefill_tensor_parallelism: 1 # Extracted from p.tp1 +pd_decode_tensor_parallelism: 4 # Extracted from d.tp4 +pd_scheduler_config: high-throughput +``` + +### Artifact Directory Structure +``` +ARTIFACT_DIR/ +├── llmd_test/ +│ ├── __test_labels__.yaml +│ ├── artifacts/ +│ │ ├── llminferenceservice.yaml +│ │ ├── endpoint.url +│ │ └── results/ +│ │ └── benchmark_short/ +├── prepare_llmd_run_pd_d_x2_p_tp1_d_tp4_p_x8_meta_llama_instruct/ +└── ... +``` + +### LLMISVC Manifest Annotations +P/D deployments include profile metadata in annotations: + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + namespace: forge-llm-d + annotations: + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 +spec: + replicas: 2 # decode pods + prefill: + replicas: 8 # prefill pods + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' # prefill tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + template: # decode template + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' # decode tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' +``` + +## Implementation Details + +### Template Profile Configuration +P/D template profiles are defined in `orchestration/config.d/deployments.yaml`: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME + pd_config: + prefill_pods: FROM_NAME + decode_pods: FROM_NAME + scheduler_config: "high-throughput" +``` + +### Global P/D Configuration +P/D-specific VLLM args, environment variables, and resources: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Parameter Extraction +The `_extract_value_from_profile_name()` function extracts values using regex patterns: + +```python +# For tensor_parallelism (main container) +match = re.search(r"d\.tp(\d+)", profile_name) # d.tp4 → 4 +if not match: + match = re.search(r"(? my-pd-preset.yaml << EOF +runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 + benchmark_key: short +EOF + +llm_d ci --preset my-pd-preset +``` + +### Matrix P/D Testing +```bash +# Use PR comment for matrix testing: +# /var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +# /var runtime.model_name: [Qwen/Qwen3-0.6B, meta-llama/Llama-3.1-8B-Instruct] +# /var runtime.benchmark_key: short + +llm_d ci # Will use PR directive overrides +``` + +### Custom P/D with Different GPU Allocation +```bash +# Test asymmetric GPU allocation: many small prefill pods, few large decode pods +# Profile: pd-d.x1-p.tp1-d.tp8-p.x16 +# 1 decode pod with 8 GPUs, 16 prefill pods with 1 GPU each + +cat > asymmetric-pd.yaml << EOF +runtime: + model_name: meta-llama/Llama-3.1-8B-Instruct + deployment_profile: pd-d.x1-p.tp1-d.tp8-p.x16 + benchmark_key: concurrent-1k-1k +EOF + +llm_d ci --preset asymmetric-pd +``` + +### Validating P/D Profile Names +```bash +# Test profile name parsing +python3 -c " +from projects.llm_d.orchestration.runtime_config import _extract_value_from_profile_name + +profile = 'pd-d.x2-p.tp1-d.tp4-p.x8' +print(f'Decode pods: {_extract_value_from_profile_name(profile, \"decode_pods\")}') +print(f'Prefill pods: {_extract_value_from_profile_name(profile, \"prefill_pods\")}') +print(f'Decode TP: {_extract_value_from_profile_name(profile, \"tensor_parallelism\")}') +print(f'Prefill TP: {_extract_value_from_profile_name(profile, \"prefill_tensor_parallelism\")}') +" +``` + +This P/D testing system provides flexible, template-based configuration with automatic parameter extraction from profile names, enabling sophisticated testing of Prefill/Decode configurations at scale. diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index cc74273d8..9c8d1b8ff 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -16,3 +16,45 @@ precise-prefix-cache: distributed-default: scheduler: {} + +# P/D (Prefill/Decode) deployment profiles +pd-small: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 2 + decode_pods: 4 + scheduler_config: "high-throughput" + +pd-medium: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 4 + decode_pods: 8 + scheduler_config: "balanced" + +pd-large: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + pd_config: + prefill_pods: 8 + decode_pods: 16 + scheduler_config: "low-latency" + +# New flavor configurations +pd-x2-ptp4-px1-dtp4: + scheduler_manifest: manifests/deployments/pd-scheduler.yaml + tensor_parallelism: 4 + pd_config: + prefill_pods: 1 # px1 + decode_pods: 4 # dtp4 + scheduler_config: "high-throughput" + +intelligentrouting-tp4-x4: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml + tensor_parallelism: 4 + replicas: 4 + scheduler: {} # scheduler enabled with default config + +simple-tp4-x4: + tensor_parallelism: 4 + replicas: 4 + # No scheduler_manifest means scheduler disabled diff --git a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml new file mode 100644 index 000000000..8d6c7e2cd --- /dev/null +++ b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml @@ -0,0 +1,86 @@ +scheduler: + template: + containers: + - name: main + env: + - name: TOKENIZER_CACHE_DIR + value: /tmp/tokenizer-cache + - name: HF_HOME + value: /tmp/tokenizer-cache + - name: TRANSFORMERS_CACHE + value: /tmp/tokenizer-cache + - name: XDG_CACHE_HOME + value: /tmp + # P/D specific environment variables + - name: PD_PREFILL_PODS + value: "{{ deployment_profile.pd_config.prefill_pods }}" + - name: PD_DECODE_PODS + value: "{{ deployment_profile.pd_config.decode_pods }}" + - name: PD_SCHEDULER_CONFIG + value: "{{ deployment_profile.pd_config.scheduler_config }}" + args: + - --cert-path + - /var/run/kserve/tls + - --pool-group + - inference.networking.x-k8s.io + - --pool-name + - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" + - --pool-namespace + - "{{ .ObjectMeta.Namespace }}" + - --zap-encoder + - json + - --grpc-port + - "9002" + - --grpc-health-port + - "9003" + - --secure-serving + - --model-server-metrics-scheme + - https + - --config-text + - | + apiVersion: inference.networking.x-k8s.io/v1alpha1 + kind: EndpointPickerConfig + plugins: + - type: pd-prefill-scheduler + config: + prefillPods: {{ deployment_profile.pd_config.prefill_pods }} + schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" + - type: pd-decode-scheduler + config: + decodePods: {{ deployment_profile.pd_config.decode_pods }} + schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" + - type: queue-scorer + - type: kv-cache-utilization-scorer + schedulingProfiles: + - name: prefill-profile + plugins: + - pluginRef: pd-prefill-scheduler + weight: 5 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - name: decode-profile + plugins: + - pluginRef: pd-decode-scheduler + weight: 5 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + - name: default + plugins: + - pluginRef: queue-scorer + weight: 2 + - pluginRef: kv-cache-utilization-scorer + weight: 2 + volumeMounts: + - name: tokenizer-cache + mountPath: /tmp/tokenizer-cache + - name: cachi2-cache + mountPath: /cachi2 + volumes: + - name: tokenizer-cache + emptyDir: {} + - name: cachi2-cache + emptyDir: {} diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 34d0609e2..169664efc 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -45,19 +45,110 @@ benchmark-short: runtime.smoke_request_key: default runtime.benchmark_key: short -gpt-oss-120b-inference-scheduling-release: - runtime.selected_preset: gpt-oss-120b-inference-scheduling-release +# Base gpt-oss configuration that can be extended +gpt-oss: runtime.model_name: openai/gpt-oss-120b runtime.deployment_profile: distributed-default runtime.smoke_request_key: default + model_cache.pvc.size: 300Gi + workloads.args: + request_type: text_completions + +gpt-oss-120b-inference-scheduling-release: + extends: + - gpt-oss + runtime.selected_preset: gpt-oss-120b-inference-scheduling-release runtime.benchmark_key: - concurrent-1k-1k - heavy-heterogeneous - multi-turn + +cks: + extends: + - smoke + +# P/D deployment profile presets +deployment-pd-small: + runtime.selected_preset: deployment-pd-small + runtime.deployment_profile: pd-small + +deployment-pd-medium: + runtime.selected_preset: deployment-pd-medium + runtime.deployment_profile: pd-medium + +deployment-pd-large: + runtime.selected_preset: deployment-pd-large + runtime.deployment_profile: pd-large + +# P/D testing presets +pd-smoke: + extends: + - smoke-base + - deployment-pd-small + runtime.selected_preset: pd-smoke + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-smoke-medium: + extends: + - smoke-base + - deployment-pd-medium + runtime.selected_preset: pd-smoke-medium + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-smoke-large: + extends: + - smoke-base + - deployment-pd-large + runtime.selected_preset: pd-smoke-large + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +pd-matrix: + runtime.selected_preset: pd-matrix + runtime.model_name: + - meta-llama/Llama-3.1-8B-Instruct + - Qwen/Qwen3-0.6B + runtime.deployment_profile: + - pd-small + - pd-medium + - pd-large + runtime.benchmark_key: + - short + - concurrent-1k-1k + runtime.smoke_request_key: default + platform.gateway.status_address_name: gateway-internal + +pd-scale-test: + runtime.selected_preset: pd-scale-test + runtime.model_name: openai/gpt-oss-120b + runtime.deployment_profile: + - pd-small + - pd-medium + - pd-large + runtime.benchmark_key: heavy-heterogeneous + runtime.smoke_request_key: default model_cache.pvc.size: 300Gi workloads.args: request_type: text_completions -cks: +# New presets ported from other tool +pd-flavors: extends: - - smoke + - gpt-oss + runtime.selected_preset: pd-flavors + runtime.deployment_profile: + - pd-x2-ptp4-px1-dtp4 + - intelligentrouting-tp4-x4 + - simple-tp4-x4 + +guidellm_multiturn_eval: + runtime.selected_preset: guidellm_multiturn_eval + runtime.benchmark_key: guidellm_multiturn_eval + runtime.smoke_request_key: default + +guidellm_heterogeneous_eval: + runtime.selected_preset: guidellm_heterogeneous_eval + runtime.benchmark_key: guidellm_heterogeneous_eval + runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index dabbe22af..c9f1c5513 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -299,6 +299,35 @@ def get_deployment_profile() -> dict[str, Any]: return resolved_profile +def get_pd_config() -> dict[str, Any]: + """Get P/D configuration from active deployment profile.""" + profile = get_deployment_profile() + return profile.get("pd_config", {}) + + +def get_prefill_pod_count() -> int: + """Get number of prefill pods for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("prefill_pods", 1) + + +def get_decode_pod_count() -> int: + """Get number of decode pods for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("decode_pods", 1) + + +def get_scheduler_config() -> str: + """Get scheduler configuration for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("scheduler_config", "default") + + +def is_pd_deployment() -> bool: + """Check if current deployment profile is a P/D deployment.""" + return bool(get_pd_config()) + + def get_smoke_request() -> dict[str, Any]: """Get the smoke request configuration""" smoke_request_key = config.project.get_config("runtime.smoke_request_key") From 545b173e61900ab6c7fde23cdeb7507afc4eb4da Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 16 Jul 2026 17:36:33 +0200 Subject: [PATCH 0171/1072] [llm_d] orchestration: render_inference_service: add support for P/D --- .../orchestration/render_inference_service.py | 186 +++++++++++++++--- projects/llm_d/orchestration/test_phase.py | 2 + 2 files changed, 165 insertions(+), 23 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index e823a867d..6a522b7f6 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -1,6 +1,5 @@ from __future__ import annotations -import copy import hashlib from pathlib import Path from typing import Any @@ -24,11 +23,15 @@ def render_inference_service_from_parts( model_slug: str, deployment_profile: dict[str, Any], model_cache: dict[str, Any], + deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render an llm_d-owned LLMInferenceService manifest from concrete runtime inputs.""" template_path = Path(config_dir) / inference_service["template"] manifest = _load_yaml(template_path) + # Check if this is a P/D deployment + is_pd_deployment = "pd_config" in deployment_profile + name = inference_service["name"] manifest["metadata"]["name"] = name manifest["metadata"]["namespace"] = namespace @@ -78,32 +81,13 @@ def render_inference_service_from_parts( "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), } - manifest["spec"]["replicas"] = deployment_profile["replicas"] manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug - serving_container = manifest["spec"]["template"]["containers"][0] - serving_container["resources"] = _build_serving_resources(deployment_profile) - if deployment_profile.get("serving_image"): - serving_container["image"] = deployment_profile["serving_image"] - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args - - scheduler = deployment_profile.get("scheduler", {}) - if scheduler is None: - manifest["spec"]["router"].pop("scheduler", None) + if is_pd_deployment: + return _render_pd_deployment(manifest, deployment_profile, deployment_profile_name) else: - manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) - if deployment_profile.get("router_image"): - manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( - deployment_profile["router_image"] - ) - - return manifest + return _render_standard_deployment(manifest, deployment_profile) def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: @@ -142,3 +126,159 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: prefix = f"--{option_name}=" bare = f"--{option_name}" return any(arg == bare or arg.startswith(prefix) for arg in args) + + +def _render_standard_deployment( + manifest: dict[str, Any], deployment_profile: dict[str, Any] +) -> dict[str, Any]: + """Render standard (non-P/D) deployment configuration.""" + manifest["spec"]["replicas"] = deployment_profile["replicas"] + + serving_container = manifest["spec"]["template"]["containers"][0] + serving_container["resources"] = _build_serving_resources(deployment_profile) + if deployment_profile.get("serving_image"): + serving_container["image"] = deployment_profile["serving_image"] + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + if vllm_args: + serving_container["args"] = vllm_args + + scheduler = deployment_profile.get("scheduler", {}) + if scheduler is None: + manifest["spec"]["router"].pop("scheduler", None) + else: + manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) + if deployment_profile.get("router_image"): + manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( + deployment_profile["router_image"] + ) + + return manifest + + +def _render_pd_deployment( + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, +) -> dict[str, Any]: + """Render P/D (Prefill/Decode) deployment configuration.""" + pd_config = deployment_profile["pd_config"] + + # Update service name to include deployment profile for P/D + current_name = manifest["metadata"]["name"] + if current_name == "llm-d" and deployment_profile_name: # Only modify if it's the default name + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + + # Set main replicas to 2 for P/D + manifest["spec"]["replicas"] = 2 + + # Configure prefill section + manifest["spec"]["prefill"] = { + "replicas": 2, + "template": _build_pd_pod_template(deployment_profile, is_prefill=True), + } + + # Configure main template (decode) + manifest["spec"]["template"] = _build_pd_pod_template(deployment_profile, is_prefill=False) + + # Simplified router for P/D + manifest["spec"]["router"] = { + "scheduler": {"template": {"containers": [{"name": "main"}]}}, + "route": {}, + "gateway": {}, + } + + return manifest + + +def _build_pd_pod_template( + deployment_profile: dict[str, Any], is_prefill: bool = False +) -> dict[str, Any]: + """Build pod template for P/D deployment.""" + tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + + # Build VLLM environment variable configuration specific to prefill/decode + if is_prefill: + # Prefill configuration + vllm_args_list = [ + "--disable-uvicorn-access-log", + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", # Appears twice in prefill + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len=40960", + ] + else: + # Decode configuration + vllm_args_list = [ + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", # Space instead of equals for decode + ] + + vllm_additional_args = " ".join(vllm_args_list) + + container = { + "name": "main", + "env": [ + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, + { + "name": "VLLM_NIXL_SIDE_CHANNEL_HOST", + "valueFrom": {"fieldRef": {"apiVersion": "v1", "fieldPath": "status.podIP"}}, + }, + {"name": "UCX_IB_GID_INDEX", "value": "3"}, + ], + "livenessProbe": { + "failureThreshold": 1000, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "initialDelaySeconds": 900, + "periodSeconds": 60, + "timeoutSeconds": 60, + }, + "readinessProbe": { + "failureThreshold": 10000, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "initialDelaySeconds": 60, + "periodSeconds": 30, + "successThreshold": 1, + "timeoutSeconds": 30, + }, + "resources": { + "limits": {"dra.llm-d.io/gpu-nic-pair": tensor_parallelism}, + "requests": { + "cpu": "4", + "dra.llm-d.io/gpu-nic-pair": tensor_parallelism, + "memory": "64Gi", + }, + }, + "securityContext": {"capabilities": {"add": ["IPC_LOCK", "SYS_RAWIO"]}}, + "startupProbe": { + "failureThreshold": 150, + "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 1, + }, + } + + template = {"serviceAccountName": "llm-d-privileged", "containers": [container]} + + # Add tolerations only to main template (decode), not prefill + if not is_prefill: + template["tolerations"] = [ + {"effect": "NoSchedule", "key": "nvidia.com/gpu", "operator": "Exists"} + ] + + return template diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2a68c2070..0fa44b436 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -249,6 +249,7 @@ def _build_inference_service_manifest() -> Path: deployment_profile = runtime_config.deep_merge(deployment_profile, benchmark_overrides) # Build the InferenceService manifest + deployment_profile_name = runtime_config.get_deployment_profile_name() manifest = render_inference_service_from_parts( config_dir=config_dir, namespace=namespace, @@ -257,6 +258,7 @@ def _build_inference_service_manifest() -> Path: model_slug=model_slug, deployment_profile=deployment_profile, model_cache=model_cache, + deployment_profile_name=deployment_profile_name, ) # Write the manifest to artifacts From b6cf2c87ea6d2156669b6e552e49ce273a566e78 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 17 Jul 2026 12:46:46 +0200 Subject: [PATCH 0172/1072] [llm_d] orchestration: work on the P/D deployment --- .../orchestration/config.d/deployments.yaml | 3 + .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../orchestration/presets.d/presets.yaml | 38 +++ .../orchestration/render_inference_service.py | 270 +++++++++++++++--- .../llm_d/orchestration/runtime_config.py | 18 +- 5 files changed, 288 insertions(+), 42 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 9c8d1b8ff..85cb9100c 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -43,6 +43,7 @@ pd-large: pd-x2-ptp4-px1-dtp4: scheduler_manifest: manifests/deployments/pd-scheduler.yaml tensor_parallelism: 4 + use_defaults: true pd_config: prefill_pods: 1 # px1 decode_pods: 4 # dtp4 @@ -52,9 +53,11 @@ intelligentrouting-tp4-x4: scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml tensor_parallelism: 4 replicas: 4 + use_defaults: true scheduler: {} # scheduler enabled with default config simple-tp4-x4: tensor_parallelism: 4 replicas: 4 + use_defaults: true # No scheduler_manifest means scheduler disabled diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 95ac92ebb..d125a4bd9 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -6,3 +6,4 @@ benchmark_key: null job_name: null namespace_override: null kserve_dry_run: false +kserve_use_kueue: false diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 169664efc..d8af640ac 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -133,6 +133,44 @@ pd-scale-test: workloads.args: request_type: text_completions +# New deployment profiles +deployment-intelligentrouting-tp4-x4: + runtime.selected_preset: deployment-intelligentrouting-tp4-x4 + runtime.deployment_profile: intelligentrouting-tp4-x4 + +deployment-simple-tp4-x4: + runtime.selected_preset: deployment-simple-tp4-x4 + runtime.deployment_profile: simple-tp4-x4 + +deployment-pd-x2-ptp4-px1-dtp4: + runtime.selected_preset: deployment-pd-x2-ptp4-px1-dtp4 + runtime.deployment_profile: pd-x2-ptp4-px1-dtp4 + +# Smoke presets for new profiles +smoke-intelligentrouting: + extends: + - smoke-base + - deployment-intelligentrouting-tp4-x4 + runtime.selected_preset: smoke-intelligentrouting + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +smoke-simple-tp4: + extends: + - smoke-base + - deployment-simple-tp4-x4 + runtime.selected_preset: smoke-simple-tp4 + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + +smoke-pd-tp4: + extends: + - smoke-base + - deployment-pd-x2-ptp4-px1-dtp4 + runtime.selected_preset: smoke-pd-tp4 + runtime.benchmark_key: short + platform.gateway.status_address_name: gateway-internal + # New presets ported from other tool pd-flavors: extends: diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 6a522b7f6..a31b4fdb5 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import hashlib from pathlib import Path from typing import Any @@ -7,6 +8,7 @@ import yaml from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.library import config def _load_yaml(path: Path) -> Any: @@ -85,9 +87,18 @@ def render_inference_service_from_parts( manifest["spec"]["model"]["name"] = model_slug if is_pd_deployment: - return _render_pd_deployment(manifest, deployment_profile, deployment_profile_name) + rendered_manifest = _render_pd_deployment( + manifest, deployment_profile, deployment_profile_name + ) else: - return _render_standard_deployment(manifest, deployment_profile) + rendered_manifest = _render_standard_deployment( + manifest, deployment_profile, deployment_profile_name + ) + + # Apply Kueue configuration if enabled + _apply_kueue_configuration(rendered_manifest) + + return rendered_manifest def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: @@ -129,32 +140,114 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: def _render_standard_deployment( - manifest: dict[str, Any], deployment_profile: dict[str, Any] + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render standard (non-P/D) deployment configuration.""" + # Check if this is intelligent routing (scheduler_manifest exists) + scheduler = deployment_profile.get("scheduler") + has_scheduler_manifest = "scheduler_manifest" in deployment_profile + is_intelligent_routing = scheduler is not None and has_scheduler_manifest + + # Update service name for intelligent routing or use_defaults + current_name = manifest["metadata"]["name"] + use_defaults = deployment_profile.get("use_defaults", False) + if ( + current_name == "llm-d" + and deployment_profile_name + and (is_intelligent_routing or use_defaults) + ): + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + manifest["spec"]["replicas"] = deployment_profile["replicas"] serving_container = manifest["spec"]["template"]["containers"][0] serving_container["resources"] = _build_serving_resources(deployment_profile) if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + + # Configure VLLM args based on deployment type and use_defaults flag tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args + use_defaults = deployment_profile.get("use_defaults", False) - scheduler = deployment_profile.get("scheduler", {}) - if scheduler is None: + if use_defaults: + # Use VLLM_ADDITIONAL_ARGS environment variable when use_defaults is set + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + + vllm_additional_args = " ".join(vllm_args) + + # Add environment variable (don't set generic env vars or args) + if "env" not in serving_container: + serving_container["env"] = [] + serving_container["env"].append( + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} + ) + elif is_intelligent_routing: + # Use environment variables for intelligent routing + vllm_args_list = [ + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", + ] + vllm_additional_args = " ".join(vllm_args_list) + + # Add or update environment variables + if "env" not in serving_container: + serving_container["env"] = [] + serving_container["env"].append( + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} + ) + else: + # Use CLI args for basic deployments + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + if vllm_args: + serving_container["args"] = vllm_args + + # Configure router/scheduler + has_scheduler_key = "scheduler" in deployment_profile + is_simple_deployment = not has_scheduler_key and not has_scheduler_manifest + + if is_simple_deployment: + # Simple deployments (no scheduler key, no scheduler_manifest) have no router section at all + manifest["spec"].pop("router", None) + elif scheduler is None: + # Some deployments might have router but no scheduler manifest["spec"]["router"].pop("scheduler", None) else: + # Configure scheduler for intelligent routing manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) if deployment_profile.get("router_image"): manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( deployment_profile["router_image"] ) + # Enhance scheduler for intelligent routing + if is_intelligent_routing: + # Ensure template structure exists + if "template" not in manifest["spec"]["router"]["scheduler"]: + manifest["spec"]["router"]["scheduler"]["template"] = { + "containers": [{"name": "main"}] + } + + scheduler_template = manifest["spec"]["router"]["scheduler"]["template"] + + # Add nodeSelector and serviceAccountName (if not already set by runtime_config.py) + if "nodeSelector" not in scheduler_template: + scheduler_template["nodeSelector"] = { + "nvidia.com/gpu.deploy.container-toolkit": "true" + } + if "serviceAccountName" not in scheduler_template: + scheduler_template["serviceAccountName"] = "llm-d-privileged" + return manifest @@ -164,7 +257,6 @@ def _render_pd_deployment( deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" - pd_config = deployment_profile["pd_config"] # Update service name to include deployment profile for P/D current_name = manifest["metadata"]["name"] @@ -198,37 +290,45 @@ def _build_pd_pod_template( ) -> dict[str, Any]: """Build pod template for P/D deployment.""" tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - - # Build VLLM environment variable configuration specific to prefill/decode - if is_prefill: - # Prefill configuration - vllm_args_list = [ - "--disable-uvicorn-access-log", - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", # Appears twice in prefill - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len=40960", - ] + use_defaults = deployment_profile.get("use_defaults", False) + + # Build VLLM environment variable configuration + if use_defaults: + # Use deployment profile's vllm_args when use_defaults is set + vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) + if not _has_cli_arg(vllm_args, "tensor-parallel-size"): + vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + vllm_additional_args = " ".join(vllm_args) else: - # Decode configuration - vllm_args_list = [ - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", # Space instead of equals for decode - ] - - vllm_additional_args = " ".join(vllm_args_list) + # Use P/D-specific configuration when use_defaults is false + if is_prefill: + # Prefill configuration + vllm_args_list = [ + "--disable-uvicorn-access-log", + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", # Appears twice in prefill + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len=40960", + ] + else: + # Decode configuration + vllm_args_list = [ + "--block-size 128", + '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', + f"--tensor-parallel-size={tensor_parallelism}", + "--disable-uvicorn-access-log", + "--enable-prefix-caching", + "--uvicorn-log-level=debug", + "--trust-remote-code", + "--gpu-memory-utilization=0.92", + "--max-model-len 40960", # Space instead of equals for decode + ] + vllm_additional_args = " ".join(vllm_args_list) container = { "name": "main", @@ -282,3 +382,91 @@ def _build_pd_pod_template( ] return template + + +def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: + """Apply Kueue annotations and labels to the ISVC manifest. + + Based on the implementation from topsail's test_llmd.py. + Can be enabled by setting runtime.kserve_use_kueue config. + """ + # Check if kueue annotations should be enabled + enable_kueue = config.project.get_config("runtime.kserve_use_kueue", False) + + if not enable_kueue: + return + + # Default kueue configuration (can be made configurable later) + kueue_config = { + "enabled": True, + "prefix": "kueue.x-k8s.io/", + "labels": {"queue-name": "default-queue"}, + "annotations": {"queue-name": "default-queue"}, + } + + # Get prefix for kueue labels/annotations + kueue_prefix = kueue_config.get("prefix", "kueue.x-k8s.io/") + + # Ensure metadata sections exist + if "metadata" not in manifest: + manifest["metadata"] = {} + if "labels" not in manifest["metadata"]: + manifest["metadata"]["labels"] = {} + if "annotations" not in manifest["metadata"]: + manifest["metadata"]["annotations"] = {} + + # Apply Kueue labels + kueue_labels = kueue_config.get("labels", {}) + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + manifest["metadata"]["labels"][full_label_key] = label_value + + # Apply Kueue annotations + kueue_annotations = kueue_config.get("annotations", {}) + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + manifest["metadata"]["annotations"][full_annotation_key] = annotation_value + + # Apply Kueue annotations to router scheduler pod template if it exists + if ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ): + scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + + # Ensure metadata exists in scheduler template + if "metadata" not in scheduler_template: + scheduler_template["metadata"] = {} + if "annotations" not in scheduler_template["metadata"]: + scheduler_template["metadata"]["annotations"] = {} + + # Apply the same Kueue annotations to the scheduler pod template + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + + # Update the scheduler template back to the data structure + manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template + + # Calculate pod group total count: 1 scheduler + number of replicas + replicas = manifest.get("spec", {}).get("replicas", 1) + + # For P/D deployments, we need to account for prefill replicas too + prefill_replicas = 0 + if "spec" in manifest and "prefill" in manifest["spec"]: + prefill_replicas = manifest["spec"]["prefill"].get("replicas", 0) + + # Total: main replicas + prefill replicas + (1 scheduler if router exists) + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) + + scheduler_count = 1 if has_scheduler else 0 + pod_group_total_count = replicas + prefill_replicas + scheduler_count + + manifest["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index c9f1c5513..f261f514e 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -294,7 +294,23 @@ def get_deployment_profile() -> dict[str, Any]: resolved_profile = deep_merge(defaults, profile) if scheduler_manifest: scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - resolved_profile = deep_merge(resolved_profile, scheduler_data) + + # When use_defaults is true, use simplified scheduler template + use_defaults = resolved_profile.get("use_defaults", False) + if use_defaults: + # Create simplified scheduler with just basic container structure + simplified_scheduler = { + "scheduler": { + "template": { + "containers": [{"name": "main"}], + "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + "serviceAccountName": "llm-d-privileged", + } + } + } + resolved_profile = deep_merge(resolved_profile, simplified_scheduler) + else: + resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile From fc629cd74e9179f569adc87cd611325476488b3b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 17 Jul 2026 14:18:19 +0200 Subject: [PATCH 0173/1072] [llm_d] tests: add reference profiles --- .../llmisvc.yaml | 62 ++++++++ .../llmisvc.yaml | 140 ++++++++++++++++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 52 +++++++ .../llm_d/tests/test_deployment_profiles.py | 137 +++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml create mode 100644 projects/llm_d/tests/test_deployment_profiles.py diff --git a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..bf220304b --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -0,0 +1,62 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-intelligentrouting-tp4-x4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + router: + scheduler: + template: + containers: + - name: main + nodeSelector: + nvidia.com/gpu.deploy.container-toolkit: 'true' + serviceAccountName: llm-d-privileged + route: {} + gateway: {} + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 diff --git a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml new file mode 100644 index 000000000..c88a7140f --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml @@ -0,0 +1,140 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-x2-ptp4-px1-dtp4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 2 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + router: + scheduler: + template: + containers: + - name: main + route: {} + gateway: {} + template: + serviceAccountName: llm-d-privileged + containers: + - name: main + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: UCX_IB_GID_INDEX + value: '3' + livenessProbe: + failureThreshold: 1000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + resources: + limits: + dra.llm-d.io/gpu-nic-pair: '4' + requests: + cpu: '4' + dra.llm-d.io/gpu-nic-pair: '4' + memory: 64Gi + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RAWIO + startupProbe: + failureThreshold: 150 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + prefill: + replicas: 2 + template: + serviceAccountName: llm-d-privileged + containers: + - name: main + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: UCX_IB_GID_INDEX + value: '3' + livenessProbe: + failureThreshold: 1000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + resources: + limits: + dra.llm-d.io/gpu-nic-pair: '4' + requests: + cpu: '4' + dra.llm-d.io/gpu-nic-pair: '4' + memory: 64Gi + securityContext: + capabilities: + add: + - IPC_LOCK + - SYS_RAWIO + startupProbe: + failureThreshold: 150 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 diff --git a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..752c71d66 --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml @@ -0,0 +1,52 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-simple-tp4-x4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model + name: qwen-qwen3-0-6b + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py new file mode 100644 index 000000000..3e0232375 --- /dev/null +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +from projects.core.library import config as core_config +from projects.core.library import env + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" +REFERENCE_DIR = Path(__file__).resolve().parent / "reference_deployments" + +# Deployment presets to test +DEPLOYMENT_PRESETS = [ + "deployment-simple-tp4-x4", + "deployment-intelligentrouting-tp4-x4", + "deployment-pd-x2-ptp4-px1-dtp4", +] + + +@pytest.fixture(autouse=True) +def _reset_project_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARTIFACT_DIR", str(tmp_path / "artifacts")) + # Unset KUBECONFIG as requested + monkeypatch.delenv("KUBECONFIG", raising=False) + env.init() + core_config.project = None + yield + core_config.project = None + + +@pytest.mark.parametrize("preset", DEPLOYMENT_PRESETS) +def test_deployment_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Test that deployment presets generate the expected LLMISVC manifests.""" + _test_preset_generates_expected_llmisvc(preset, tmp_path) + + +def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Helper function to test a preset generates the expected LLMISVC manifest.""" + # Set up environment + artifact_dir = tmp_path / "artifacts" / preset + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Create variable overrides to ensure dry-run mode + variable_overrides_path = artifact_dir / "000__ci_metadata" / "variable_overrides.yaml" + variable_overrides_path.parent.mkdir(parents=True, exist_ok=True) + variable_overrides_path.write_text( + yaml.safe_dump( + { + "runtime.kserve_dry_run": True, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + env_vars = os.environ.copy() + env_vars["ARTIFACT_DIR"] = str(artifact_dir) + env_vars.pop("KUBECONFIG", None) + + # Run the CI script with the preset + ci_script = PROJECT_ROOT / "projects" / "llm_d" / "orchestration" / "ci.py" + cmd = [str(ci_script), "--preset", preset, "test"] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + env=env_vars, + ) + + # Check that the command succeeded + if result.returncode != 0: + pytest.fail( + f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + + # Find the generated LLMISVC manifest + generated_manifest = _find_generated_llmisvc(artifact_dir) + if not generated_manifest.exists(): + pytest.fail(f"Generated LLMISVC manifest not found at {generated_manifest}") + + # Load the generated manifest + with generated_manifest.open("r", encoding="utf-8") as f: + generated_content = yaml.safe_load(f) + + # Load the reference manifest + reference_manifest = REFERENCE_DIR / preset / "llmisvc.yaml" + if not reference_manifest.exists(): + pytest.fail(f"Reference manifest not found at {reference_manifest}") + + with reference_manifest.open("r", encoding="utf-8") as f: + reference_content = yaml.safe_load(f) + + # Compare the manifests + if generated_content != reference_content: + # Generate a diff for better error reporting + import difflib + + generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) + reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) + + diff = list( + difflib.unified_diff( + reference_yaml.splitlines(keepends=True), + generated_yaml.splitlines(keepends=True), + fromfile=f"reference/{preset}/llmisvc.yaml", + tofile=f"generated/{preset}/llmisvc.yaml", + lineterm="", + ) + ) + + pytest.fail( + f"Generated LLMISVC does not match reference for preset {preset}:\n{''.join(diff)}" + ) + + +def _find_generated_llmisvc(artifact_dir: Path) -> Path: + """Find the generated LLMISVC manifest in the artifact directory.""" + # Look for the pattern: $ARTIFACT_DIR/**/deploy_llmisvc/src/llminferenceservice.yaml + deploy_llmisvc_dirs = list(artifact_dir.glob("**/*__deploy_llmisvc")) + + if not deploy_llmisvc_dirs: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") + + if len(deploy_llmisvc_dirs) > 1: + raise ValueError( + f"Multiple deploy_llmisvc directories found in {artifact_dir}: {deploy_llmisvc_dirs}" + ) + + llmisvc_path = deploy_llmisvc_dirs[0] / "src" / "llminferenceservice.yaml" + return llmisvc_path From 4badfcc16d43bcebdccfc952ca8cccfc285bce35 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 20 Jul 2026 13:47:13 +0200 Subject: [PATCH 0174/1072] [llm_d] orchestration: keep working on P/D --- .../orchestration/config.d/deployments.yaml | 52 ++-- .../llm_d/orchestration/config.d/runtime.yaml | 11 +- .../orchestration/presets.d/presets.yaml | 132 +---------- .../orchestration/render_inference_service.py | 108 +++++---- .../llm_d/orchestration/runtime_config.py | 222 +++++++++++++++++- .../llmisvc.yaml | 2 +- .../llmisvc.yaml | 4 +- .../deployment-simple-tp4-x4/llmisvc.yaml | 5 +- .../llm_d/tests/test_deployment_profiles.py | 206 ++++++++++++---- 9 files changed, 471 insertions(+), 271 deletions(-) rename projects/llm_d/tests/reference_deployments/{deployment-intelligentrouting-tp4-x4 => cpt-reference-flavors/deployment-intelligentrouting-tp4-x}/llmisvc.yaml (96%) rename projects/llm_d/tests/reference_deployments/{deployment-pd-x2-ptp4-px1-dtp4 => cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1}/llmisvc.yaml (97%) rename projects/llm_d/tests/reference_deployments/{ => cpt-reference-flavors}/deployment-simple-tp4-x4/llmisvc.yaml (93%) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 85cb9100c..e643c56c1 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -17,47 +17,25 @@ precise-prefix-cache: distributed-default: scheduler: {} -# P/D (Prefill/Decode) deployment profiles -pd-small: +# Template flavor configurations with name-based extraction +template__pd: scheduler_manifest: manifests/deployments/pd-scheduler.yaml + tensor_parallelism: FROM_NAME + use_kserve_defaults: true pd_config: - prefill_pods: 2 - decode_pods: 4 + prefill_pods: FROM_NAME + decode_pods: FROM_NAME scheduler_config: "high-throughput" -pd-medium: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - pd_config: - prefill_pods: 4 - decode_pods: 8 - scheduler_config: "balanced" - -pd-large: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - pd_config: - prefill_pods: 8 - decode_pods: 16 - scheduler_config: "low-latency" - -# New flavor configurations -pd-x2-ptp4-px1-dtp4: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - tensor_parallelism: 4 - use_defaults: true - pd_config: - prefill_pods: 1 # px1 - decode_pods: 4 # dtp4 - scheduler_config: "high-throughput" - -intelligentrouting-tp4-x4: +template__intelligentrouting: + use_kserve_defaults: true scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml - tensor_parallelism: 4 - replicas: 4 - use_defaults: true + tensor_parallelism: FROM_NAME + replicas: FROM_NAME scheduler: {} # scheduler enabled with default config -simple-tp4-x4: - tensor_parallelism: 4 - replicas: 4 - use_defaults: true - # No scheduler_manifest means scheduler disabled +template__simple: + tensor_parallelism: FROM_NAME + replicas: FROM_NAME + use_kserve_defaults: true + scheduler: null # scheduler disabled diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index d125a4bd9..73b527cf8 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,5 +5,12 @@ smoke_request_key: default benchmark_key: null job_name: null namespace_override: null -kserve_dry_run: false -kserve_use_kueue: false +kserve_dry_run: true +kueue: + enable: true + queue_name: default-queue +router: + template: + file: manifests/scheduler_config/__router_template__.yaml + placeholder: __ENDPOINT_PICKER_CONFIG__ + epp: manifests/scheduler_config diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index d8af640ac..d8ef284ee 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -1,15 +1,12 @@ __multiple: true deployment-approximate-prefix-cache: - runtime.selected_preset: deployment-approximate-prefix-cache runtime.deployment_profile: approximate-prefix-cache deployment-precise-prefix-cache: - runtime.selected_preset: deployment-precise-prefix-cache runtime.deployment_profile: precise-prefix-cache deployment-distributed-default: - runtime.selected_preset: deployment-distributed-default runtime.deployment_profile: distributed-default smoke-base: @@ -21,7 +18,6 @@ smoke: extends: - smoke-base - deployment-approximate-prefix-cache - runtime.selected_preset: smoke runtime.benchmark_key: short platform.gateway.status_address_name: gateway-internal @@ -29,23 +25,19 @@ smoke-precise: extends: - smoke-base - deployment-precise-prefix-cache - runtime.selected_preset: smoke-precise smoke-default-scheduler: extends: - smoke-base - deployment-distributed-default - runtime.selected_preset: smoke-default-scheduler benchmark-short: extends: - deployment-approximate-prefix-cache - runtime.selected_preset: benchmark-short runtime.model_name: meta-llama/Llama-3.1-8B-Instruct runtime.smoke_request_key: default runtime.benchmark_key: short -# Base gpt-oss configuration that can be extended gpt-oss: runtime.model_name: openai/gpt-oss-120b runtime.deployment_profile: distributed-default @@ -57,136 +49,26 @@ gpt-oss: gpt-oss-120b-inference-scheduling-release: extends: - gpt-oss - runtime.selected_preset: gpt-oss-120b-inference-scheduling-release runtime.benchmark_key: - concurrent-1k-1k - heavy-heterogeneous - multi-turn -cks: - extends: - - smoke - -# P/D deployment profile presets -deployment-pd-small: - runtime.selected_preset: deployment-pd-small - runtime.deployment_profile: pd-small - -deployment-pd-medium: - runtime.selected_preset: deployment-pd-medium - runtime.deployment_profile: pd-medium - -deployment-pd-large: - runtime.selected_preset: deployment-pd-large - runtime.deployment_profile: pd-large - -# P/D testing presets -pd-smoke: - extends: - - smoke-base - - deployment-pd-small - runtime.selected_preset: pd-smoke - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-smoke-medium: - extends: - - smoke-base - - deployment-pd-medium - runtime.selected_preset: pd-smoke-medium - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-smoke-large: - extends: - - smoke-base - - deployment-pd-large - runtime.selected_preset: pd-smoke-large - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -pd-matrix: - runtime.selected_preset: pd-matrix - runtime.model_name: - - meta-llama/Llama-3.1-8B-Instruct - - Qwen/Qwen3-0.6B - runtime.deployment_profile: - - pd-small - - pd-medium - - pd-large - runtime.benchmark_key: - - short - - concurrent-1k-1k - runtime.smoke_request_key: default - platform.gateway.status_address_name: gateway-internal - -pd-scale-test: - runtime.selected_preset: pd-scale-test - runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: - - pd-small - - pd-medium - - pd-large - runtime.benchmark_key: heavy-heterogeneous - runtime.smoke_request_key: default - model_cache.pvc.size: 300Gi - workloads.args: - request_type: text_completions - -# New deployment profiles -deployment-intelligentrouting-tp4-x4: - runtime.selected_preset: deployment-intelligentrouting-tp4-x4 +# Test deployment profiles +test-intelligentrouting: runtime.deployment_profile: intelligentrouting-tp4-x4 -deployment-simple-tp4-x4: - runtime.selected_preset: deployment-simple-tp4-x4 +test-simple: runtime.deployment_profile: simple-tp4-x4 -deployment-pd-x2-ptp4-px1-dtp4: - runtime.selected_preset: deployment-pd-x2-ptp4-px1-dtp4 - runtime.deployment_profile: pd-x2-ptp4-px1-dtp4 - -# Smoke presets for new profiles -smoke-intelligentrouting: - extends: - - smoke-base - - deployment-intelligentrouting-tp4-x4 - runtime.selected_preset: smoke-intelligentrouting - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -smoke-simple-tp4: - extends: - - smoke-base - - deployment-simple-tp4-x4 - runtime.selected_preset: smoke-simple-tp4 - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal - -smoke-pd-tp4: - extends: - - smoke-base - - deployment-pd-x2-ptp4-px1-dtp4 - runtime.selected_preset: smoke-pd-tp4 - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal +test-pd: + runtime.deployment_profile: pd-d.x2-p.tp4-d.tp4-p.x1 # New presets ported from other tool -pd-flavors: +cpt-reference-flavors: extends: - gpt-oss - runtime.selected_preset: pd-flavors runtime.deployment_profile: - - pd-x2-ptp4-px1-dtp4 + - pd-d.x2-p.tp4-d.tp4-p.x1 - intelligentrouting-tp4-x4 - simple-tp4-x4 - -guidellm_multiturn_eval: - runtime.selected_preset: guidellm_multiturn_eval - runtime.benchmark_key: guidellm_multiturn_eval - runtime.smoke_request_key: default - -guidellm_heterogeneous_eval: - runtime.selected_preset: guidellm_heterogeneous_eval - runtime.benchmark_key: guidellm_heterogeneous_eval - runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index a31b4fdb5..ba510a273 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -16,6 +16,44 @@ def _load_yaml(path: Path) -> Any: return yaml.safe_load(handle) +def _create_model_cache_spec( + model_cache: dict[str, Any], + source_uri: str, + source_scheme: str, + model_slug: str, + namespace: str, +) -> dict[str, Any] | None: + """Create model cache specification if caching is enabled and applicable.""" + if not model_cache.get("enabled", False) or source_uri.startswith(("pvc://", "pvc+hf://")): + return None + + pvc_defaults = model_cache["pvc"] + pvc_prefix = pvc_defaults["name_prefix"] + cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] + pvc_name = truncate_k8s_name( + f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" + ) + model_path = pvc_defaults["model_directory_name"] + + return { + "source_uri": source_uri, + "source_scheme": source_scheme, + "cache_key": cache_key, + "namespace": namespace, + "pvc_name": pvc_name, + "pvc_size": pvc_defaults["size"], + "access_mode": pvc_defaults["access_mode"], + "storage_class_name": pvc_defaults.get("storage_class_name"), + "model_path": model_path, + "model_uri": f"pvc://{pvc_name}/{model_path}", + "marker_filename": model_cache["marker_filename"], + "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", + "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), + "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), + "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), + } + + def render_inference_service_from_parts( *, config_dir: str | Path, @@ -55,33 +93,13 @@ def render_inference_service_from_parts( source_uri = f"hf://{model_name}" source_scheme = "hf" - cache_spec = None - if model_cache.get("enabled", False) and not source_uri.startswith(("pvc://", "pvc+hf://")): - pvc_defaults = model_cache["pvc"] - pvc_prefix = pvc_defaults["name_prefix"] - cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] - pvc_name = truncate_k8s_name( - f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" - ) - model_path = pvc_defaults["model_directory_name"] - - cache_spec = { - "source_uri": source_uri, - "source_scheme": source_scheme, - "cache_key": cache_key, - "namespace": namespace, - "pvc_name": pvc_name, - "pvc_size": pvc_defaults["size"], - "access_mode": pvc_defaults["access_mode"], - "storage_class_name": pvc_defaults.get("storage_class_name"), - "model_path": model_path, - "model_uri": f"pvc://{pvc_name}/{model_path}", - "marker_filename": model_cache["marker_filename"], - "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", - "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), - "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), - "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), - } + cache_spec = _create_model_cache_spec( + model_cache=model_cache, + source_uri=source_uri, + source_scheme=source_scheme, + model_slug=model_slug, + namespace=namespace, + ) manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug @@ -150,15 +168,10 @@ def _render_standard_deployment( has_scheduler_manifest = "scheduler_manifest" in deployment_profile is_intelligent_routing = scheduler is not None and has_scheduler_manifest - # Update service name for intelligent routing or use_defaults + # Update service name for intelligent routing or use_kserve_defaults current_name = manifest["metadata"]["name"] - use_defaults = deployment_profile.get("use_defaults", False) - if ( - current_name == "llm-d" - and deployment_profile_name - and (is_intelligent_routing or use_defaults) - ): - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" manifest["spec"]["replicas"] = deployment_profile["replicas"] @@ -167,12 +180,12 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - # Configure VLLM args based on deployment type and use_defaults flag + # Configure VLLM args based on deployment type and use_kserve_defaults flag tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_defaults = deployment_profile.get("use_defaults", False) + use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) - if use_defaults: - # Use VLLM_ADDITIONAL_ARGS environment variable when use_defaults is set + if use_kserve_defaults: + # Use VLLM_ADDITIONAL_ARGS environment variable when use_kserve_defaults is set vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) if not _has_cli_arg(vllm_args, "tensor-parallel-size"): vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") @@ -290,17 +303,17 @@ def _build_pd_pod_template( ) -> dict[str, Any]: """Build pod template for P/D deployment.""" tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_defaults = deployment_profile.get("use_defaults", False) + use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) # Build VLLM environment variable configuration - if use_defaults: - # Use deployment profile's vllm_args when use_defaults is set + if use_kserve_defaults: + # Use deployment profile's vllm_args when use_kserve_defaults is set vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) if not _has_cli_arg(vllm_args, "tensor-parallel-size"): vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") vllm_additional_args = " ".join(vllm_args) else: - # Use P/D-specific configuration when use_defaults is false + # Use P/D-specific configuration when use_kserve_defaults is false if is_prefill: # Prefill configuration vllm_args_list = [ @@ -391,17 +404,18 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: Can be enabled by setting runtime.kserve_use_kueue config. """ # Check if kueue annotations should be enabled - enable_kueue = config.project.get_config("runtime.kserve_use_kueue", False) + enable_kueue = config.project.get_config("runtime.kueue.enabled") if not enable_kueue: return - # Default kueue configuration (can be made configurable later) + # Configure kueue settings + queue_name = config.project.get_config("runtime.kueue.queue_name") kueue_config = { "enabled": True, "prefix": "kueue.x-k8s.io/", - "labels": {"queue-name": "default-queue"}, - "annotations": {"queue-name": "default-queue"}, + "labels": {"queue-name": queue_name}, + "annotations": {"queue-name": queue_name}, } # Get prefix for kueue labels/annotations diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f261f514e..f2cb4939a 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -114,11 +114,166 @@ def deep_merge(base: Any, override: Any) -> Any: return merged +def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: + """Extract configuration values from profile names. + + Supports patterns: + - simple-tp4-x4: tensor_parallelism=4, replicas=4 + - intelligentrouting-tp4-x4: tensor_parallelism=4, replicas=4 + - pd-d.x2-p.tp4-d.tp4-p.x1: prefill_pods=1, decode_pods=2, tensor_parallelism=4 + + Args: + profile_name: The deployment profile name + field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods) + + Returns: + Extracted integer value + + Raises: + ValueError: If the pattern is not recognized or value cannot be extracted + """ + # P/D pattern: pd-d.x2-p.tp4-d.tp4-p.x1 + if profile_name.startswith("pd-"): + # P/D naming: pd-d.x{decode_pods}-p.tp{tensor_parallelism}-d.tp{decode_tensor_parallelism}-p.x{prefill_pods} + # Extract different components + if field_name == "tensor_parallelism": + # Look for p.tp4 pattern (prefill tensor_parallelism) + match = re.search(r"p\.tp(\d+)", profile_name) + if match: + return int(match.group(1)) + # Fallback: look for any tp pattern + match = re.search(r"tp(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "prefill_pods": + # Look for p.x1 pattern + match = re.search(r"p\.x(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "decode_pods": + # Look for d.x2 pattern + match = re.search(r"d\.x(\d+)", profile_name) + if match: + return int(match.group(1)) + else: + # Standard patterns: simple-tp4-x4, intelligentrouting-tp4-x4 + if field_name == "tensor_parallelism": + # Look for tp4 pattern + match = re.search(r"tp(\d+)", profile_name) + if match: + return int(match.group(1)) + elif field_name == "replicas": + # Look for x4 pattern + match = re.search(r"x(\d+)", profile_name) + if match: + return int(match.group(1)) + + raise ValueError(f"Could not extract {field_name} from profile name: {profile_name}") + + +def _resolve_from_name_values(profile_name: str, profile_data: dict[str, Any]) -> dict[str, Any]: + """Resolve FROM_NAME placeholders in profile configuration with values from profile name. + + Args: + profile_name: The deployment profile name to extract values from + profile_data: The profile configuration that may contain FROM_NAME placeholders + + Returns: + Profile configuration with FROM_NAME placeholders resolved to actual values + """ + resolved = copy.deepcopy(profile_data) + + def resolve_value(obj: Any, path: str = "") -> Any: + if isinstance(obj, dict): + result = {} + for key, value in obj.items(): + current_path = f"{path}.{key}" if path else key + result[key] = resolve_value(value, current_path) + return result + elif isinstance(obj, list): + return [resolve_value(item, path) for item in obj] + elif obj == "FROM_NAME": + # Determine which field to extract based on the path + field_name = path.split(".")[-1] # Get the last part of the path + try: + return _extract_value_from_profile_name(profile_name, field_name) + except ValueError as e: + logger.warning(f"Failed to extract {field_name} from {profile_name}: {e}") + return obj # Return the original FROM_NAME if extraction fails + else: + return obj + + return resolve_value(resolved) + + def _load_yaml(path: Path) -> Any: with path.open(encoding="utf-8") as handle: return yaml.safe_load(handle) +def _load_scheduler_with_epp_config( + config_dir: Path, scheduler_manifest: str, deployment_profile: dict[str, Any] +) -> dict[str, Any]: + """Load scheduler configuration using router template with EPP config replacement. + + Uses configurable paths from runtime.router config. + + Args: + config_dir: Configuration directory path + scheduler_manifest: Path to EPP config file (e.g., "manifests/deployments/approximate-prefix-cache.yaml") + deployment_profile: Resolved deployment profile for templating + + Returns: + Scheduler data with EPP config content replacing placeholder + """ + + # Get configurable paths and placeholder + router_template_file = template_config.get("runtime.router.template.file") + router_placeholder = template_config.get("runtime.router.template.placeholder") + epp_config_dir = router_config.get("runtime.router.template.epp") + + # Load the router template + router_template_path = config_dir / router_template_file + if not router_template_path.exists(): + # Fallback to legacy behavior if template doesn't exist + raise FileNotFoundError( + f"Router template not found at {router_template_path}" + ) + + # Load the EPP config content + # Convert scheduler_manifest path to EPP config directory + # e.g., "manifests/deployments/approximate-prefix-cache.yaml" -> "manifests/scheduler_config/approximate-prefix-cache.yaml" + manifest_path = Path(scheduler_manifest) + epp_config_filename = manifest_path.name + epp_config_path = config_dir / epp_config_dir / epp_config_filename + + if not epp_config_path.exists(): + raise FileNotFoundError( + f"EPP config not found at {epp_config_path}" + ) + + # Load both files + router_template_data = _load_yaml(router_template_path) + epp_config_data = _load_yaml(epp_config_path) + + # Replace placeholder in the parsed data structure (avoids YAML formatting issues) + def replace_placeholder_in_structure(obj): + if isinstance(obj, dict): + return {key: replace_placeholder_in_structure(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [replace_placeholder_in_structure(item) for item in obj] + elif isinstance(obj, str) and obj == router_placeholder: + # Convert EPP config to YAML string for the --config-text argument + return yaml.dump(epp_config_data, default_flow_style=False, sort_keys=False).strip() + else: + return obj + + # Perform the replacement in the structure + replaced_data = replace_placeholder_in_structure(router_template_data) + + return replaced_data + + def get_config_dir() -> Path: """Get the LLM-D configuration directory""" return ORCHESTRATION_DIR @@ -130,8 +285,8 @@ def get_job_name() -> str: if job_name: return job_name - preset_name = config.project.get_config("runtime.selected_preset") - return f"local-{preset_name}" + deployment_profile = config.project.get_config("runtime.deployment_profile") + return f"local-{deployment_profile}" def get_platform_config() -> dict[str, Any]: @@ -284,32 +439,78 @@ def get_deployment_profile_name() -> str: return deployment_profiles[0] +def _resolve_template_profile( + profile_name: str, deployment_config: dict[str, Any] +) -> dict[str, Any]: + """Resolve a profile name to a template-based configuration if no direct match exists. + + Args: + profile_name: The deployment profile name (e.g., "simple-tp4-x4") + deployment_config: The full deployments configuration + + Returns: + Template configuration to use, or raises ValueError if no template matches + + Raises: + ValueError: If no template pattern matches the profile name + """ + # Try to match profile name patterns to templates + if profile_name.startswith("pd-"): + template_name = "template__pd" + elif "intelligentrouting" in profile_name: + template_name = "template__intelligentrouting" + elif profile_name.startswith("simple-") or re.search(r"-tp\d+-x\d+$", profile_name): + template_name = "template__simple" + else: + raise ValueError(f"No template found for profile name pattern: {profile_name}") + + if template_name not in deployment_config: + raise ValueError(f"Template {template_name} not found in deployment config") + + logger.info(f"Using template {template_name} for profile {profile_name}") + return copy.deepcopy(deployment_config[template_name]) + + def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() deployment_config = copy.deepcopy(config.project.get_config("deployments")) - profile = copy.deepcopy(deployment_config[profile_name]) + + # First try direct profile lookup + if profile_name in deployment_config: + profile = copy.deepcopy(deployment_config[profile_name]) + else: + # Fall back to template-based resolution + profile = _resolve_template_profile(profile_name, deployment_config) + defaults = copy.deepcopy(deployment_config.get("defaults", {})) scheduler_manifest = profile.pop("scheduler_manifest", None) resolved_profile = deep_merge(defaults, profile) - if scheduler_manifest: - scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - # When use_defaults is true, use simplified scheduler template - use_defaults = resolved_profile.get("use_defaults", False) - if use_defaults: + # Resolve FROM_NAME placeholders with values extracted from profile name + resolved_profile = _resolve_from_name_values(profile_name, resolved_profile) + + if scheduler_manifest: + RENAME THIS TO use_rhoai_defaults + # When use_kserve_defaults is true, use simplified scheduler template + use_kserve_defaults = resolved_profile.get("use_kserve_defaults", False) + if use_kserve_defaults: # Create simplified scheduler with just basic container structure simplified_scheduler = { "scheduler": { "template": { "containers": [{"name": "main"}], + # force the scheduler to run on a GPU node. 'gpu.present: true' not working on AKS/CKS "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, - "serviceAccountName": "llm-d-privileged", } } } resolved_profile = deep_merge(resolved_profile, simplified_scheduler) else: + # When use_kserve_defaults is false, use __router_template__.yaml with EPP config replacement + scheduler_data = _load_scheduler_with_epp_config( + get_config_dir(), scheduler_manifest, resolved_profile + ) resolved_profile = deep_merge(resolved_profile, scheduler_data) return resolved_profile @@ -341,7 +542,8 @@ def get_scheduler_config() -> str: def is_pd_deployment() -> bool: """Check if current deployment profile is a P/D deployment.""" - return bool(get_pd_config()) + profile = get_deployment_profile() + return "pd_config" in profile def get_smoke_request() -> dict[str, Any]: diff --git a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml similarity index 96% rename from projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml index bf220304b..7761ef972 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-intelligentrouting-tp4-x4 - namespace: forge-llm-d + namespace: forge-llm-d-qwen-qwen3-0-6b-intelligentrouting-tp4-x annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics diff --git a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml similarity index 97% rename from projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml index c88a7140f..1618f4d63 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-pd-x2-ptp4-px1-dtp4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml @@ -1,8 +1,8 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-pd-x2-ptp4-px1-dtp4 - namespace: forge-llm-d + name: llm-d-pd-d.x2-p.tp4-d.tp4-p.x1 + namespace: forge-llm-d-qwen-qwen3-0-6b-pd-d-x2-p-tp4-d-tp4-p-x1 annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics diff --git a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml similarity index 93% rename from projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 752c71d66..0c1c9bbfd 100644 --- a/projects/llm_d/tests/reference_deployments/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-simple-tp4-x4 - namespace: forge-llm-d + namespace: forge-llm-d-qwen-qwen3-0-6b-simple-tp4-x4 annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics @@ -15,6 +15,9 @@ spec: model: uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model name: qwen-qwen3-0-6b + router: + route: {} + gateway: {} template: tolerations: - key: nvidia.com/gpu diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 3e0232375..f17952695 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -16,11 +16,12 @@ # Deployment presets to test DEPLOYMENT_PRESETS = [ - "deployment-simple-tp4-x4", - "deployment-intelligentrouting-tp4-x4", - "deployment-pd-x2-ptp4-px1-dtp4", + "cpt-reference-flavors", ] +# Check for save deployments mode via environment variable +SAVE_DEPLOYMENTS = os.environ.get("SAVE_DEPLOYMENTS", "false").lower() in ("true", "1", "yes") + @pytest.fixture(autouse=True) def _reset_project_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): @@ -77,61 +78,174 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): # Check that the command succeeded if result.returncode != 0: pytest.fail( - f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" ) - # Find the generated LLMISVC manifest - generated_manifest = _find_generated_llmisvc(artifact_dir) - if not generated_manifest.exists(): - pytest.fail(f"Generated LLMISVC manifest not found at {generated_manifest}") - - # Load the generated manifest - with generated_manifest.open("r", encoding="utf-8") as f: - generated_content = yaml.safe_load(f) - - # Load the reference manifest - reference_manifest = REFERENCE_DIR / preset / "llmisvc.yaml" - if not reference_manifest.exists(): - pytest.fail(f"Reference manifest not found at {reference_manifest}") - - with reference_manifest.open("r", encoding="utf-8") as f: - reference_content = yaml.safe_load(f) - - # Compare the manifests - if generated_content != reference_content: - # Generate a diff for better error reporting - import difflib - - generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) - reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) - - diff = list( - difflib.unified_diff( - reference_yaml.splitlines(keepends=True), - generated_yaml.splitlines(keepends=True), - fromfile=f"reference/{preset}/llmisvc.yaml", - tofile=f"generated/{preset}/llmisvc.yaml", - lineterm="", - ) - ) + # Find all generated LLMISVC manifests + deployments = _find_generated_llmisvcs(artifact_dir) + if not deployments: pytest.fail( - f"Generated LLMISVC does not match reference for preset {preset}:\n{''.join(diff)}" + f"No generated LLMISVC manifests found in {artifact_dir}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" ) + # If saving deployments, save them and skip comparison + if SAVE_DEPLOYMENTS: + _save_deployments(preset, deployments) + pytest.skip(f"Saved {len(deployments)} deployments for preset {preset}") + return -def _find_generated_llmisvc(artifact_dir: Path) -> Path: - """Find the generated LLMISVC manifest in the artifact directory.""" + # Test each deployment against its reference + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + print(f"Testing deployment for profile: {profile_name}") + + if not generated_manifest.exists(): + pytest.fail( + f"Generated LLMISVC manifest not found at {generated_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + # Load the generated manifest + with generated_manifest.open("r", encoding="utf-8") as f: + generated_content = yaml.safe_load(f) + + # Determine reference manifest path + # Look in preset-specific directory structure: preset/deployment-{profile_name}/llmisvc.yaml + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + if not reference_manifest.exists(): + pytest.fail( + f"Reference manifest not found at {reference_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + with reference_manifest.open("r", encoding="utf-8") as f: + reference_content = yaml.safe_load(f) + + # Compare the manifests + if generated_content != reference_content: + # Generate a diff for better error reporting + import difflib + + generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) + reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) + + diff = list( + difflib.unified_diff( + reference_yaml.splitlines(keepends=True), + generated_yaml.splitlines(keepends=True), + fromfile=f"reference/{preset}/deployment-{profile_name}/llmisvc.yaml", + tofile=f"generated/{preset}/{profile_name}/llmisvc.yaml", + lineterm="", + ) + ) + + pytest.fail( + f"Generated LLMISVC does not match reference for profile {profile_name} in preset {preset}:\n{''.join(diff)}\n\n" + f"💡 To update reference deployments with new output, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + +def _save_deployments(preset: str, deployments: list[dict[str, Path]]): + """Save generated deployments as reference manifests.""" + print(f"\nSaving {len(deployments)} deployments for preset {preset}:") + + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + # Create reference directory for this preset and deployment profile + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_deployment_dir.mkdir(parents=True, exist_ok=True) + + # Copy the generated manifest to the reference location + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + with generated_manifest.open("r", encoding="utf-8") as f: + content = f.read() + + with reference_manifest.open("w", encoding="utf-8") as f: + f.write(content) + + print(f" Saved {profile_name} → {reference_manifest}") + + print(f"Reference deployments saved to: {reference_preset_dir}") + + +def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: + """Find all generated LLMISVC manifests in the artifact directory. + + Returns: + List of dictionaries with 'profile_name' and 'manifest_path' keys + """ # Look for the pattern: $ARTIFACT_DIR/**/deploy_llmisvc/src/llminferenceservice.yaml deploy_llmisvc_dirs = list(artifact_dir.glob("**/*__deploy_llmisvc")) if not deploy_llmisvc_dirs: raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") - if len(deploy_llmisvc_dirs) > 1: + deployments = [] + for deploy_dir in deploy_llmisvc_dirs: + llmisvc_path = deploy_dir / "src" / "llminferenceservice.yaml" + if llmisvc_path.exists(): + # Extract profile name from the directory structure + # Pattern: .../__llmd_run_{profile_name}_{model_slug}/.../__deploy_llmisvc + parent_path = ( + deploy_dir.parent.parent.name + ) # e.g., "001__llmd_run_pd-d-x2-p-tp4-d-tp4-p-x1_qwen-qwen3-0-6b" + if "__llmd_run_" in parent_path: + # Extract the part after __llmd_run_ + run_part = parent_path.split("__llmd_run_", 1)[1] + + # Split by underscore to separate profile from model + parts = run_part.split("_") + + # Find where the model name starts (models typically start with specific patterns) + model_start_idx = len(parts) + for i, part in enumerate(parts): + if part.startswith(("qwen", "meta", "openai", "llama")): + model_start_idx = i + break + + # Profile name is everything before the model + if model_start_idx > 0: + profile_parts = parts[:model_start_idx] + profile_name = "-".join(profile_parts) + else: + # Fallback: use entire run part minus likely model + profile_name = "-".join(parts[:-1]) if len(parts) > 1 else parts[0] + + deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) + else: + # Fallback: use directory name + deployments.append({"profile_name": "unknown", "manifest_path": llmisvc_path}) + + return deployments + + +def _find_generated_llmisvc(artifact_dir: Path) -> Path: + """Find the generated LLMISVC manifest in the artifact directory (legacy single deployment).""" + deployments = _find_generated_llmisvcs(artifact_dir) + + if len(deployments) == 1: + return deployments[0]["manifest_path"] + elif len(deployments) > 1: raise ValueError( - f"Multiple deploy_llmisvc directories found in {artifact_dir}: {deploy_llmisvc_dirs}" + f"Multiple deploy_llmisvc directories found in {artifact_dir}. " + f"Use _find_generated_llmisvcs() instead for multi-deployment presets." ) - - llmisvc_path = deploy_llmisvc_dirs[0] / "src" / "llminferenceservice.yaml" - return llmisvc_path + else: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") From 75274092f8696e8509e20531ab643bcb966d1a3a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 20 Jul 2026 14:51:39 +0200 Subject: [PATCH 0175/1072] [llm_d] orchestration: manifests: scheduler_config: unify --- .../manifests/deployments/pd-scheduler.yaml | 86 ------------------- .../deployments/precise-prefix-cache.yaml | 73 ---------------- .../__router_template__.yaml} | 19 +--- .../approximate-prefix-cache.yaml | 15 ++++ .../precise-prefix-cache.yaml | 30 +++++++ 5 files changed, 48 insertions(+), 175 deletions(-) delete mode 100644 projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml delete mode 100644 projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml rename projects/llm_d/orchestration/manifests/{deployments/approximate-prefix-cache.yaml => scheduler_config/__router_template__.yaml} (66%) create mode 100644 projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml create mode 100644 projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml diff --git a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml b/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml deleted file mode 100644 index 8d6c7e2cd..000000000 --- a/projects/llm_d/orchestration/manifests/deployments/pd-scheduler.yaml +++ /dev/null @@ -1,86 +0,0 @@ -scheduler: - template: - containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - # P/D specific environment variables - - name: PD_PREFILL_PODS - value: "{{ deployment_profile.pd_config.prefill_pods }}" - - name: PD_DECODE_PODS - value: "{{ deployment_profile.pd_config.decode_pods }}" - - name: PD_SCHEDULER_CONFIG - value: "{{ deployment_profile.pd_config.scheduler_config }}" - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: pd-prefill-scheduler - config: - prefillPods: {{ deployment_profile.pd_config.prefill_pods }} - schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" - - type: pd-decode-scheduler - config: - decodePods: {{ deployment_profile.pd_config.decode_pods }} - schedulerStrategy: "{{ deployment_profile.pd_config.scheduler_config }}" - - type: queue-scorer - - type: kv-cache-utilization-scorer - schedulingProfiles: - - name: prefill-profile - plugins: - - pluginRef: pd-prefill-scheduler - weight: 5 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - name: decode-profile - plugins: - - pluginRef: pd-decode-scheduler - weight: 5 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} diff --git a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml deleted file mode 100644 index 7e4e4ff1a..000000000 --- a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml +++ /dev/null @@ -1,73 +0,0 @@ -scheduler: - template: - containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. - # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with - # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead - # of the older precise-prefix-cache-scorer-only configuration kept here. - - type: single-profile-handler - - type: precise-prefix-cache-scorer - parameters: - indexerConfig: - tokenProcessorConfig: - blockSize: 64 - hashSeed: "42" - tokenizersPoolConfig: - hf: - tokenizersCacheDir: /tmp/tokenizers - - type: kv-cache-utilization-scorer - - type: queue-scorer - - type: max-score-picker - schedulingProfiles: - - name: default - plugins: - - pluginRef: precise-prefix-cache-scorer - weight: 3.0 - - pluginRef: kv-cache-utilization-scorer - weight: 2.0 - - pluginRef: queue-scorer - weight: 2.0 - - pluginRef: max-score-picker - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} diff --git a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml similarity index 66% rename from projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml rename to projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml index ae62e49d4..8c33cbd30 100644 --- a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml +++ b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml @@ -30,22 +30,9 @@ scheduler: - --model-server-metrics-scheme - https - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: prefix-cache-scorer - schedulingProfiles: - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 + + - __ENDPOINT_PICKER_CONFIG__ + volumeMounts: - name: tokenizer-cache mountPath: /tmp/tokenizer-cache diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml new file mode 100644 index 000000000..ea0cf650c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml @@ -0,0 +1,15 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: prefix-cache-scorer +schedulingProfiles: + - name: default + plugins: + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: prefix-cache-scorer + weight: 3.0 diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml new file mode 100644 index 000000000..20e8895bf --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml @@ -0,0 +1,30 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. + # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with + # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead + # of the older precise-prefix-cache-scorer-only configuration kept here. + - type: single-profile-handler + - type: precise-prefix-cache-scorer + parameters: + indexerConfig: + tokenProcessorConfig: + blockSize: 64 + hashSeed: "42" + tokenizersPoolConfig: + hf: + tokenizersCacheDir: /tmp/tokenizers + - type: kv-cache-utilization-scorer + - type: queue-scorer + - type: max-score-picker +schedulingProfiles: + - name: default + plugins: + - pluginRef: precise-prefix-cache-scorer + weight: 3.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: max-score-picker From c76f03871de5008a5b2597a57b84ee976df9a81a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:27:40 +0200 Subject: [PATCH 0176/1072] [core] library: env: create link from /tmp/forge_last to the current ARTIFACT_DIR --- projects/core/library/env.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 7c07ca732..05ea5eed1 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -103,6 +103,12 @@ def init(daily_artifact_dir=False): artifact_dir.mkdir(parents=True, exist_ok=True) os.environ["ARTIFACT_DIR"] = str(artifact_dir) + # Create or update forge_last symlink + forge_last_link = env_forge_base_dir / "forge_last" + if forge_last_link.exists() or forge_last_link.is_symlink(): + forge_last_link.unlink() + forge_last_link.symlink_to(artifact_dir.name) + artifact_dir.mkdir(parents=True, exist_ok=True) # Set BASE_ARTIFACT_DIR to the initial value (immutable) From a4cbb648f6d0b1ab89b1d067a21763c14db661fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:29:00 +0200 Subject: [PATCH 0177/1072] [core] library: env: use '%Y%m%d-%H%M-%S' if '%Y%m%d-%H%M' already exists --- projects/core/library/env.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 05ea5eed1..4def14e1e 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -98,7 +98,14 @@ def init(daily_artifact_dir=False): else: env_forge_base_dir = pathlib.Path(os.environ.get("FORGE_BASE_DIR", "/tmp")) - artifact_dir = env_forge_base_dir / f"forge_{time.strftime('%Y%m%d-%H%M')}" + # Try base format first, add seconds if directory exists + base_name = f"forge_{time.strftime('%Y%m%d-%H%M')}" + artifact_dir = env_forge_base_dir / base_name + + if artifact_dir.exists(): + # Directory exists, add seconds to make it unique + unique_name = f"forge_{time.strftime('%Y%m%d-%H%M-%S')}" + artifact_dir = env_forge_base_dir / unique_name artifact_dir.mkdir(parents=True, exist_ok=True) os.environ["ARTIFACT_DIR"] = str(artifact_dir) From 50dd143cb5ad4f6c1fea806687823e1243e14e23 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:56:52 +0200 Subject: [PATCH 0178/1072] [llm_d] work on P/D --- .../orchestration/config.d/deployments.yaml | 77 +++-- .../orchestration/config.d/platform.yaml | 4 - .../llm_d/orchestration/config.d/project.yaml | 3 - .../llm_d/orchestration/config.d/runtime.yaml | 6 +- .../manifests/llminferenceservice.yaml | 40 +-- .../llm_d/orchestration/presets.d/cpt.yaml | 10 + .../orchestration/presets.d/presets.yaml | 19 -- .../orchestration/render_inference_service.py | 293 +++++++----------- .../llm_d/orchestration/runtime_config.py | 216 +++++-------- .../llmisvc.yaml | 10 +- .../llmisvc.yaml | 140 --------- .../llmisvc.yaml | 69 +++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 12 +- .../llm_d/tests/test_deployment_profiles.py | 43 +-- 14 files changed, 341 insertions(+), 601 deletions(-) delete mode 100644 projects/llm_d/orchestration/config.d/project.yaml create mode 100644 projects/llm_d/orchestration/presets.d/cpt.yaml rename projects/llm_d/tests/reference_deployments/cpt-reference-flavors/{deployment-intelligentrouting-tp4-x => deployment-intelligentrouting-tp4-x4}/llmisvc.yaml (83%) delete mode 100644 projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml create mode 100644 projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index e643c56c1..f25bdfdf8 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -1,41 +1,56 @@ defaults: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d replicas: 1 tensor_parallelism: 1 - vllm_args: - - --max-model-len=4096 - - --gpu-memory-utilization=0.92 - - --trust-remote-code - - --no-enable-log-requests - - --enable-prefix-caching + vllm_extra: + args: + - --max-model-len=4096 + - --gpu-memory-utilization=0.92 + - --trust-remote-code + - --no-enable-log-requests + - --enable-prefix-caching -approximate-prefix-cache: - scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' -precise-prefix-cache: - scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml +profiles: + approximate-prefix-cache: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml -distributed-default: - scheduler: {} + precise-prefix-cache: + scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml + + distributed-default: + scheduler: {} # Template flavor configurations with name-based extraction -template__pd: - scheduler_manifest: manifests/deployments/pd-scheduler.yaml - tensor_parallelism: FROM_NAME - use_kserve_defaults: true - pd_config: - prefill_pods: FROM_NAME - decode_pods: FROM_NAME - scheduler_config: "high-throughput" +templates: + pd: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME + pd_config: + prefill_pods: FROM_NAME + decode_pods: FROM_NAME -template__intelligentrouting: - use_kserve_defaults: true - scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml - tensor_parallelism: FROM_NAME - replicas: FROM_NAME - scheduler: {} # scheduler enabled with default config + intelligentrouting: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME + replicas: FROM_NAME -template__simple: - tensor_parallelism: FROM_NAME - replicas: FROM_NAME - use_kserve_defaults: true - scheduler: null # scheduler disabled + simple: + scheduler_manifest: null # scheduler disabled + tensor_parallelism: FROM_NAME + replicas: FROM_NAME diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 32325769a..8f592e46f 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -1,9 +1,5 @@ cluster: minimum_openshift_version: "4.19.9" - namespace: - name: forge-llm-d - prefix: llm-d - max_length: 63 gpu_node_label_selector: nvidia.com/gpu.present=true nfd_gpu_detection_labels: - feature.node.kubernetes.io/pci-10de.present diff --git a/projects/llm_d/orchestration/config.d/project.yaml b/projects/llm_d/orchestration/config.d/project.yaml deleted file mode 100644 index 4d6b0b88f..000000000 --- a/projects/llm_d/orchestration/config.d/project.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: llm_d -args: - - smoke diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 73b527cf8..c2f2dd14e 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -4,10 +4,10 @@ deployment_profile: approximate-prefix-cache smoke_request_key: default benchmark_key: null job_name: null -namespace_override: null -kserve_dry_run: true +namespace: forge-llm-d +kserve_dry_run: false kueue: - enable: true + enabled: false queue_name: default-queue router: template: diff --git a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml index cff616f84..c3043bf31 100644 --- a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml +++ b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml @@ -16,45 +16,7 @@ spec: scheduler: template: containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} + - name: main nodeSelector: nvidia.com/gpu.present: "true" route: {} diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml new file mode 100644 index 000000000..dfb731711 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -0,0 +1,10 @@ +__multiple: true + +# CPT reference profiles +cpt-reference-flavors: + extends: + - gpt-oss + runtime.deployment_profile: + - pd-d.x2-p.tp1-d.tp4-p.x8 + - intelligentrouting-tp4-x4 + - simple-tp4-x4 diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index d8ef284ee..491296ed8 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -53,22 +53,3 @@ gpt-oss-120b-inference-scheduling-release: - concurrent-1k-1k - heavy-heterogeneous - multi-turn - -# Test deployment profiles -test-intelligentrouting: - runtime.deployment_profile: intelligentrouting-tp4-x4 - -test-simple: - runtime.deployment_profile: simple-tp4-x4 - -test-pd: - runtime.deployment_profile: pd-d.x2-p.tp4-d.tp4-p.x1 - -# New presets ported from other tool -cpt-reference-flavors: - extends: - - gpt-oss - runtime.deployment_profile: - - pd-d.x2-p.tp4-d.tp4-p.x1 - - intelligentrouting-tp4-x4 - - simple-tp4-x4 diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index ba510a273..79e433ee0 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -77,12 +77,16 @@ def render_inference_service_from_parts( manifest["metadata"]["namespace"] = namespace manifest["metadata"].setdefault("labels", {}) manifest["metadata"]["labels"].update( - { - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - } + config.project.get_config("deployments.defaults.labels") or {} ) + # Add deployment profile name as annotation for testing + manifest["metadata"].setdefault("annotations", {}) + if deployment_profile_name: + manifest["metadata"]["annotations"]["forge.openshift.io/deployment-profile"] = ( + deployment_profile_name + ) + if model_name.startswith("oci://"): source_uri = model_name source_scheme = "oci" @@ -120,13 +124,13 @@ def render_inference_service_from_parts( def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + tensor_parallelism = deployment_profile["tensor_parallelism"] profile_resources = deployment_profile.get("resources", {}) rendered_resources: dict[str, Any] = {} for bound in ("requests", "limits"): source = profile_resources.get(bound, {}) - rendered_bound = {"nvidia.com/gpu": tensor_parallelism} + rendered_bound = {"nvidia.com/gpu": str(tensor_parallelism)} for resource_name in ("cpu", "memory"): value = source.get(resource_name) if value not in (None, ""): @@ -157,6 +161,26 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: return any(arg == bare or arg.startswith(prefix) for arg in args) +def _build_vllm_additional_args( + deployment_profile: dict[str, Any], +) -> str: + """Build VLLM_ADDITIONAL_ARGS string based on deployment profile configuration. + + Args: + deployment_profile: The deployment profile configuration + + Returns: + String suitable for VLLM_ADDITIONAL_ARGS environment variable + """ + + vllm_default_args = _build_vllm_args( + config.project.get_config("deployments.defaults.vllm_extra.args") + ) + vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra.args", {})) + + return " ".join(vllm_default_args + vllm_deploy_args) + + def _render_standard_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], @@ -166,10 +190,6 @@ def _render_standard_deployment( # Check if this is intelligent routing (scheduler_manifest exists) scheduler = deployment_profile.get("scheduler") has_scheduler_manifest = "scheduler_manifest" in deployment_profile - is_intelligent_routing = scheduler is not None and has_scheduler_manifest - - # Update service name for intelligent routing or use_kserve_defaults - current_name = manifest["metadata"]["name"] manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" @@ -180,50 +200,13 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - # Configure VLLM args based on deployment type and use_kserve_defaults flag - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) + vllm_additional_args = _build_vllm_additional_args(deployment_profile) - if use_kserve_defaults: - # Use VLLM_ADDITIONAL_ARGS environment variable when use_kserve_defaults is set - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") + # Add environment variable (don't set generic env vars or args) + if "env" not in serving_container: + serving_container["env"] = [] - vllm_additional_args = " ".join(vllm_args) - - # Add environment variable (don't set generic env vars or args) - if "env" not in serving_container: - serving_container["env"] = [] - serving_container["env"].append( - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} - ) - elif is_intelligent_routing: - # Use environment variables for intelligent routing - vllm_args_list = [ - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", - ] - vllm_additional_args = " ".join(vllm_args_list) - - # Add or update environment variables - if "env" not in serving_container: - serving_container["env"] = [] - serving_container["env"].append( - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args} - ) - else: - # Use CLI args for basic deployments - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args + serving_container["env"].append({"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}) # Configure router/scheduler has_scheduler_key = "scheduler" in deployment_profile @@ -243,24 +226,6 @@ def _render_standard_deployment( deployment_profile["router_image"] ) - # Enhance scheduler for intelligent routing - if is_intelligent_routing: - # Ensure template structure exists - if "template" not in manifest["spec"]["router"]["scheduler"]: - manifest["spec"]["router"]["scheduler"]["template"] = { - "containers": [{"name": "main"}] - } - - scheduler_template = manifest["spec"]["router"]["scheduler"]["template"] - - # Add nodeSelector and serviceAccountName (if not already set by runtime_config.py) - if "nodeSelector" not in scheduler_template: - scheduler_template["nodeSelector"] = { - "nvidia.com/gpu.deploy.container-toolkit": "true" - } - if "serviceAccountName" not in scheduler_template: - scheduler_template["serviceAccountName"] = "llm-d-privileged" - return manifest @@ -270,131 +235,111 @@ def _render_pd_deployment( deployment_profile_name: str | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" + from .runtime_config import get_decode_pod_count, get_prefill_pod_count - # Update service name to include deployment profile for P/D - current_name = manifest["metadata"]["name"] - if current_name == "llm-d" and deployment_profile_name: # Only modify if it's the default name - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" - - # Set main replicas to 2 for P/D - manifest["spec"]["replicas"] = 2 + # Set manifest name with deployment profile + manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" # Configure prefill section manifest["spec"]["prefill"] = { - "replicas": 2, - "template": _build_pd_pod_template(deployment_profile, is_prefill=True), + "replicas": get_prefill_pod_count(), + "template": _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=True + ), } # Configure main template (decode) - manifest["spec"]["template"] = _build_pd_pod_template(deployment_profile, is_prefill=False) - - # Simplified router for P/D - manifest["spec"]["router"] = { - "scheduler": {"template": {"containers": [{"name": "main"}]}}, - "route": {}, - "gateway": {}, - } + manifest["spec"]["replicas"] = get_decode_pod_count() + manifest["spec"]["template"] = _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=False + ) return manifest def _build_pd_pod_template( - deployment_profile: dict[str, Any], is_prefill: bool = False + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, + is_prefill: bool = False, ) -> dict[str, Any]: """Build pod template for P/D deployment.""" - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - use_kserve_defaults = deployment_profile.get("use_kserve_defaults", False) - - # Build VLLM environment variable configuration - if use_kserve_defaults: - # Use deployment profile's vllm_args when use_kserve_defaults is set - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - vllm_additional_args = " ".join(vllm_args) + + # Get P/D extra configuration from deployments config + + pd_vllm_extra = config.project.get_config("deployments.pd.vllm_extra") + + # Build VLLM args with correct tensor parallelism + + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_vllm_args = _build_vllm_additional_args(prefill_profile) + else: + # For decode pods, use main tensor parallelism + base_vllm_args = _build_vllm_additional_args(deployment_profile) + + # Add P/D extra args + + pd_extra_args = pd_vllm_extra.get("args", []) + all_vllm_args = base_vllm_args.split() + pd_extra_args + vllm_additional_args = " ".join(all_vllm_args) + + # Build base environment variables + base_env = [ + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, + ] + + # Add P/D extra environment variables + pd_extra_env = pd_vllm_extra.get("env", []) + all_env = base_env + copy.deepcopy(pd_extra_env) + + # Build base resources with correct tensor parallelism + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_resources = _build_serving_resources(prefill_profile) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_resources = _build_serving_resources(deployment_profile) else: - # Use P/D-specific configuration when use_kserve_defaults is false - if is_prefill: - # Prefill configuration - vllm_args_list = [ - "--disable-uvicorn-access-log", - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", # Appears twice in prefill - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len=40960", - ] - else: - # Decode configuration - vllm_args_list = [ - "--block-size 128", - '--kv-transfer-config \'{"kv_connector":"NixlConnector", "kv_role":"kv_both"}\'', - f"--tensor-parallel-size={tensor_parallelism}", - "--disable-uvicorn-access-log", - "--enable-prefix-caching", - "--uvicorn-log-level=debug", - "--trust-remote-code", - "--gpu-memory-utilization=0.92", - "--max-model-len 40960", # Space instead of equals for decode - ] - vllm_additional_args = " ".join(vllm_args_list) + # For decode pods, use main tensor parallelism + base_resources = _build_serving_resources(deployment_profile) + # Add P/D extra resources to both requests and limits + pd_resources = config.project.get_config("deployments.pd.resources") + + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + base_resources[bound].update(pd_resources) + + # Build container configuration container = { "name": "main", - "env": [ - {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, - { - "name": "VLLM_NIXL_SIDE_CHANNEL_HOST", - "valueFrom": {"fieldRef": {"apiVersion": "v1", "fieldPath": "status.podIP"}}, - }, - {"name": "UCX_IB_GID_INDEX", "value": "3"}, - ], - "livenessProbe": { - "failureThreshold": 1000, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "initialDelaySeconds": 900, - "periodSeconds": 60, - "timeoutSeconds": 60, - }, - "readinessProbe": { - "failureThreshold": 10000, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "initialDelaySeconds": 60, - "periodSeconds": 30, - "successThreshold": 1, - "timeoutSeconds": 30, - }, - "resources": { - "limits": {"dra.llm-d.io/gpu-nic-pair": tensor_parallelism}, - "requests": { - "cpu": "4", - "dra.llm-d.io/gpu-nic-pair": tensor_parallelism, - "memory": "64Gi", - }, - }, - "securityContext": {"capabilities": {"add": ["IPC_LOCK", "SYS_RAWIO"]}}, - "startupProbe": { - "failureThreshold": 150, - "httpGet": {"path": "/health", "port": 8000, "scheme": "HTTPS"}, - "periodSeconds": 10, - "successThreshold": 1, - "timeoutSeconds": 1, - }, + "resources": base_resources, + "env": all_env, } - template = {"serviceAccountName": "llm-d-privileged", "containers": [container]} - - # Add tolerations only to main template (decode), not prefill - if not is_prefill: - template["tolerations"] = [ - {"effect": "NoSchedule", "key": "nvidia.com/gpu", "operator": "Exists"} - ] + # Add serving image if specified + if deployment_profile.get("serving_image"): + container["image"] = deployment_profile["serving_image"] - return template + return {"containers": [container]} def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f2cb4939a..19d3ba066 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -12,7 +12,7 @@ import yaml -from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.dsl.utils import slugify_identifier from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -32,7 +32,6 @@ class RunSpec: benchmark_slug: str | None namespace: str artifact_dirname: str - namespace_is_managed: bool def init() -> Path: @@ -124,7 +123,7 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: Args: profile_name: The deployment profile name - field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods) + field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods, prefill_tensor_parallelism) Returns: Extracted integer value @@ -137,12 +136,11 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: # P/D naming: pd-d.x{decode_pods}-p.tp{tensor_parallelism}-d.tp{decode_tensor_parallelism}-p.x{prefill_pods} # Extract different components if field_name == "tensor_parallelism": - # Look for p.tp4 pattern (prefill tensor_parallelism) - match = re.search(r"p\.tp(\d+)", profile_name) + # For main container: d.tp4 or tp4 + match = re.search(r"d\.tp(\d+)", profile_name) if match: return int(match.group(1)) - # Fallback: look for any tp pattern - match = re.search(r"tp(\d+)", profile_name) + match = re.search(r"(? int: if match: return int(match.group(1)) elif field_name == "decode_pods": - # Look for d.x2 pattern + # For main container replicas: d.x2 or x2 match = re.search(r"d\.x(\d+)", profile_name) if match: return int(match.group(1)) + match = re.search(r"(? int: if match: return int(match.group(1)) elif field_name == "replicas": - # Look for x4 pattern - match = re.search(r"x(\d+)", profile_name) + # For standard deployments: x4 (but not d.x4 or p.x4) + match = re.search(r"(? dict[str, Any]: return normalize_platform_config(copy.deepcopy(config.project.get_config("platform"))) -def _derive_base_namespace() -> str: - platform_data = get_platform_config() - namespace_override = _get_runtime_value("namespace_override") - namespace_config = platform_data["cluster"]["namespace"] - default_namespace = namespace_config.get("name") - - if namespace_override: - return namespace_override - if default_namespace: - return default_namespace - - return derive_namespace( - get_job_name(), - namespace_config["prefix"], - namespace_config["max_length"], - ) - - def get_namespace() -> str: - """Get the resolved namespace for this execution""" - return _derive_base_namespace() - - -_namespace_is_managed_override: bool | None = None - - -def get_namespace_is_managed() -> bool: - """Check if namespace is managed (auto-derived) vs explicitly configured. - - Inside activate_run_spec(), returns the base managed state captured before - the run-spec override was applied (otherwise the namespace_override set by - activate_run_spec would always flip this to False). - """ - if _namespace_is_managed_override is not None: - return _namespace_is_managed_override - - namespace_override = _get_runtime_value("namespace_override") - platform_data = get_platform_config() - default_namespace = platform_data["cluster"]["namespace"].get("name") - - return namespace_override is None and default_namespace is None + """Get the namespace for this execution""" + return _get_runtime_value("namespace") def get_model_name() -> str: @@ -454,30 +418,27 @@ def _resolve_template_profile( Raises: ValueError: If no template pattern matches the profile name """ - # Try to match profile name patterns to templates - if profile_name.startswith("pd-"): - template_name = "template__pd" - elif "intelligentrouting" in profile_name: - template_name = "template__intelligentrouting" - elif profile_name.startswith("simple-") or re.search(r"-tp\d+-x\d+$", profile_name): - template_name = "template__simple" - else: - raise ValueError(f"No template found for profile name pattern: {profile_name}") + templates = deployment_config.get("templates", {}) + if not templates: + raise ValueError("No templates section found in deployment config") - if template_name not in deployment_config: - raise ValueError(f"Template {template_name} not found in deployment config") + # Extract first component and check if it matches a template + template_name = profile_name.partition("-")[0] + if template_name not in templates: + raise ValueError(f"No template found for profile name pattern: {profile_name}") logger.info(f"Using template {template_name} for profile {profile_name}") - return copy.deepcopy(deployment_config[template_name]) + return copy.deepcopy(templates[template_name]) def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() deployment_config = copy.deepcopy(config.project.get_config("deployments")) - # First try direct profile lookup - if profile_name in deployment_config: - profile = copy.deepcopy(deployment_config[profile_name]) + # First try direct profile lookup in profiles section + profiles = deployment_config.get("profiles", {}) + if profile_name in profiles: + profile = copy.deepcopy(profiles[profile_name]) else: # Fall back to template-based resolution profile = _resolve_template_profile(profile_name, deployment_config) @@ -490,28 +451,41 @@ def get_deployment_profile() -> dict[str, Any]: # Resolve FROM_NAME placeholders with values extracted from profile name resolved_profile = _resolve_from_name_values(profile_name, resolved_profile) - if scheduler_manifest: - RENAME THIS TO use_rhoai_defaults - # When use_kserve_defaults is true, use simplified scheduler template - use_kserve_defaults = resolved_profile.get("use_kserve_defaults", False) - if use_kserve_defaults: - # Create simplified scheduler with just basic container structure - simplified_scheduler = { - "scheduler": { - "template": { - "containers": [{"name": "main"}], - # force the scheduler to run on a GPU node. 'gpu.present: true' not working on AKS/CKS - "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, - } + scheduler_content = None + if scheduler_manifest is None: + # scheduler disabled, nothing to do + return resolved_profile + + if isinstance(scheduler_manifest, dict) and len(scheduler_manifest) == 0: # {} + # Create simplified scheduler with just basic container structure - override completely + scheduler_content = { + "scheduler": { + "template": { + "containers": [{"name": "main"}], } } - resolved_profile = deep_merge(resolved_profile, simplified_scheduler) - else: - # When use_kserve_defaults is false, use __router_template__.yaml with EPP config replacement - scheduler_data = _load_scheduler_with_epp_config( - get_config_dir(), scheduler_manifest, resolved_profile - ) - resolved_profile = deep_merge(resolved_profile, scheduler_data) + } + # Don't merge - directly assign to override template defaults + resolved_profile["scheduler"] = scheduler_content["scheduler"] + else: + # use __router_template__.yaml with EPP config replacement + scheduler_content = _load_scheduler_with_epp_config( + get_config_dir(), scheduler_manifest, resolved_profile + ) + + resolved_profile = deep_merge(resolved_profile, scheduler_content) + + # force the scheduler to run on a GPU node. + # 'gpu.present: true' doesn't work on AKS/CKS + scheduler_node_selector = { + "scheduler": { + "template": { + "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + } + } + } + + resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) return resolved_profile @@ -579,29 +553,13 @@ def get_run_specs() -> list[RunSpec]: benchmark_entries = [(None, None)] combinations = list(product(model_names, profile_names, benchmark_entries)) - base_namespace = _derive_base_namespace() - namespace_max_length = get_platform_config()["cluster"]["namespace"]["max_length"] - # Compute base managed state once (ignores per-spec overrides) - namespace_is_managed = get_namespace_is_managed() + namespace = get_namespace() + run_specs: list[RunSpec] = [] for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) - if len(combinations) == 1: - namespace = base_namespace - artifact_dirname = "llmd_run" - else: - ns_parts = [base_namespace, model_slug, profile_slug] - dir_parts = ["llmd_run", profile_slug, model_slug] - if bench_slug is not None: - ns_parts.append(bench_slug) - dir_parts.append(bench_slug) - namespace = truncate_k8s_name( - "-".join(ns_parts), - max_length=namespace_max_length, - ) - artifact_dirname = "_".join(dir_parts) run_specs.append( RunSpec( @@ -612,8 +570,7 @@ def get_run_specs() -> list[RunSpec]: benchmark_key=bench_key, benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=artifact_dirname, - namespace_is_managed=namespace_is_managed, + artifact_dirname=f"llmd__{bench_key or 'default'}", ) ) @@ -623,7 +580,6 @@ def get_run_specs() -> list[RunSpec]: @contextmanager def activate_run_spec(run_spec: RunSpec): """Temporarily activate a run spec by setting its runtime config values.""" - global _namespace_is_managed_override saved = { "model_name": _get_runtime_value("model_name"), @@ -631,32 +587,19 @@ def activate_run_spec(run_spec: RunSpec): "namespace_override": _get_runtime_value("namespace_override"), "benchmark_key": _get_runtime_value("benchmark_key"), } - prev_managed_override = _namespace_is_managed_override config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) - config.project.set_config("runtime.namespace_override", run_spec.namespace) config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) - _namespace_is_managed_override = run_spec.namespace_is_managed try: yield finally: config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) - config.project.set_config("runtime.namespace_override", saved["namespace_override"]) config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) - _namespace_is_managed_override = prev_managed_override def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: - cluster = platform_data["cluster"] - if "namespace" not in cluster: - cluster["namespace"] = { - "name": cluster.pop("namespace_name", None), - "prefix": cluster.pop("namespace_prefix"), - "max_length": cluster.pop("namespace_max_length"), - } - operators = platform_data["operators"] if isinstance(operators, list): platform_data["operators"] = { @@ -669,23 +612,6 @@ def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: return platform_data -def derive_namespace(job_name: str, prefix: str, max_length: int) -> str: - slug = re.sub(r"[^a-z0-9-]+", "-", job_name.lower()) - slug = re.sub(r"-{2,}", "-", slug).strip("-") - if not slug: - slug = "run" - - if slug.startswith(f"{prefix}-"): - namespace = slug - else: - namespace = f"{prefix}-{slug}" - - namespace = namespace[:max_length].rstrip("-") - if not namespace: - raise ValueError(f"Could not derive a valid namespace from job name: {job_name}") - return namespace - - def version_tuple(value: str) -> tuple[int, ...]: numbers = re.findall(r"\d+", value) return tuple(int(number) for number in numbers[:3]) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml similarity index 83% rename from projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml rename to projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 7761ef972..6265d7c82 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -2,19 +2,20 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-intelligentrouting-tp4-x4 - namespace: forge-llm-d-qwen-qwen3-0-6b-intelligentrouting-tp4-x + namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: intelligentrouting-tp4-x4 labels: app.kubernetes.io/managed-by: forge forge.openshift.io/project: llm_d spec: replicas: 4 model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b router: scheduler: template: @@ -22,7 +23,6 @@ spec: - name: main nodeSelector: nvidia.com/gpu.deploy.container-toolkit: 'true' - serviceAccountName: llm-d-privileged route: {} gateway: {} template: @@ -59,4 +59,4 @@ spec: env: - name: VLLM_ADDITIONAL_ARGS value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + --no-enable-log-requests --enable-prefix-caching diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml deleted file mode 100644 index 1618f4d63..000000000 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d-x2-p-tp4-d-tp4-p-x1/llmisvc.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: LLMInferenceService -metadata: - name: llm-d-pd-d.x2-p.tp4-d.tp4-p.x1 - namespace: forge-llm-d-qwen-qwen3-0-6b-pd-d-x2-p-tp4-d-tp4-p-x1 - annotations: - security.opendatahub.io/enable-auth: 'false' - prometheus.io/path: /metrics - prometheus.io/port: '8000' - labels: - app.kubernetes.io/managed-by: forge - forge.openshift.io/project: llm_d -spec: - replicas: 2 - model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b - router: - scheduler: - template: - containers: - - name: main - route: {} - gateway: {} - template: - serviceAccountName: llm-d-privileged - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: UCX_IB_GID_INDEX - value: '3' - livenessProbe: - failureThreshold: 1000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 - resources: - limits: - dra.llm-d.io/gpu-nic-pair: '4' - requests: - cpu: '4' - dra.llm-d.io/gpu-nic-pair: '4' - memory: 64Gi - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RAWIO - startupProbe: - failureThreshold: 150 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Exists - prefill: - replicas: 2 - template: - serviceAccountName: llm-d-privileged - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: UCX_IB_GID_INDEX - value: '3' - livenessProbe: - failureThreshold: 1000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 - resources: - limits: - dra.llm-d.io/gpu-nic-pair: '4' - requests: - cpu: '4' - dra.llm-d.io/gpu-nic-pair: '4' - memory: 64Gi - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RAWIO - startupProbe: - failureThreshold: 150 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml new file mode 100644 index 000000000..bc538b5ce --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -0,0 +1,69 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 2 + model: + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b + router: + scheduler: + template: + containers: + - name: main + nodeSelector: + nvidia.com/gpu.present: 'true' + route: {} + gateway: {} + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + rdma/ib: '1' + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + prefill: + replicas: 8 + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' + rdma/ib: '1' + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code + --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 0c1c9bbfd..6e3edb0be 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -2,22 +2,20 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: name: llm-d-simple-tp4-x4 - namespace: forge-llm-d-qwen-qwen3-0-6b-simple-tp4-x4 + namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' prometheus.io/path: /metrics prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: simple-tp4-x4 labels: app.kubernetes.io/managed-by: forge forge.openshift.io/project: llm_d spec: replicas: 4 model: - uri: pvc://llm-d-model-qwen-qwen3-0-6b-fe62ee7991/model - name: qwen-qwen3-0-6b - router: - route: {} - gateway: {} + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b template: tolerations: - key: nvidia.com/gpu @@ -52,4 +50,4 @@ spec: env: - name: VLLM_ADDITIONAL_ARGS value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --tensor-parallel-size=4 + --no-enable-log-requests --enable-prefix-caching diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index f17952695..9c99c8d52 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -53,6 +53,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): yaml.safe_dump( { "runtime.kserve_dry_run": True, + "runtime.namespace_override": "forge-llm-d", }, sort_keys=True, ), @@ -201,37 +202,17 @@ def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: for deploy_dir in deploy_llmisvc_dirs: llmisvc_path = deploy_dir / "src" / "llminferenceservice.yaml" if llmisvc_path.exists(): - # Extract profile name from the directory structure - # Pattern: .../__llmd_run_{profile_name}_{model_slug}/.../__deploy_llmisvc - parent_path = ( - deploy_dir.parent.parent.name - ) # e.g., "001__llmd_run_pd-d-x2-p-tp4-d-tp4-p-x1_qwen-qwen3-0-6b" - if "__llmd_run_" in parent_path: - # Extract the part after __llmd_run_ - run_part = parent_path.split("__llmd_run_", 1)[1] - - # Split by underscore to separate profile from model - parts = run_part.split("_") - - # Find where the model name starts (models typically start with specific patterns) - model_start_idx = len(parts) - for i, part in enumerate(parts): - if part.startswith(("qwen", "meta", "openai", "llama")): - model_start_idx = i - break - - # Profile name is everything before the model - if model_start_idx > 0: - profile_parts = parts[:model_start_idx] - profile_name = "-".join(profile_parts) - else: - # Fallback: use entire run part minus likely model - profile_name = "-".join(parts[:-1]) if len(parts) > 1 else parts[0] - - deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) - else: - # Fallback: use directory name - deployments.append({"profile_name": "unknown", "manifest_path": llmisvc_path}) + # Read the manifest and extract profile name from annotations + with llmisvc_path.open("r", encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + ANNOTATION = "forge.openshift.io/deployment-profile" + profile_name = manifest.get("metadata", {}).get("annotations", {}).get(ANNOTATION) + + if not profile_name: + raise ValueError(f"Annotation '{ANNOTATION}' missing in {llmisvc_path}") + + deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) return deployments From 0c0f6a2022838fe154b9201e3a55823727e7307b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 13:57:38 +0200 Subject: [PATCH 0179/1072] [llm_d] CHANGELOG-orchestration: add a changelog --- projects/llm_d/CHANGELOG-orchestration.md | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/projects/llm_d/CHANGELOG-orchestration.md b/projects/llm_d/CHANGELOG-orchestration.md index 46fdeddcd..c471d93c2 100644 --- a/projects/llm_d/CHANGELOG-orchestration.md +++ b/projects/llm_d/CHANGELOG-orchestration.md @@ -1,5 +1,66 @@ # LLM-D Orchestration Changelog +## 2026-07-21 - Enhanced P/D Configuration System + +### P/D Profile Name-Based Configuration +- **FROM_NAME Parameter Extraction**: Implemented automatic parameter extraction from P/D profile names + - **Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + - **Tensor Parallelism**: Separate extraction for prefill (`p.tp4`) vs decode (`d.tp4` or `tp4`) containers + - **Pod Replicas**: Separate extraction for prefill (`p.x8`) vs decode (`d.x2` or `x2`) pods + - **Strict Parsing**: Removed fallback logic for precise parameter extraction + +### Template-Based Deployment Profiles +- **Template System**: Added `template__pd` with `FROM_NAME` placeholders for dynamic configuration + - **Auto-Resolution**: Profile names automatically populate `tensor_parallelism`, `prefill_pods`, `decode_pods` + - **Scheduler Integration**: Template profiles support simplified scheduler configuration (`scheduler_manifest: {}`) + +### Global P/D Configuration Architecture +- **Centralized P/D Config**: Moved P/D-specific settings to global `deployments.pd` configuration + - **VLLM Extra Args**: Global P/D VLLM arguments (e.g., `--block-size`, `--kv-transfer-config`) + - **Environment Variables**: Global P/D environment variables (e.g., `VLLM_NIXL_SIDE_CHANNEL_HOST`) + - **Resource Requirements**: Global P/D resource specifications (e.g., `rdma/ib: '1'`) + +### Enhanced VLLM Arguments System +- **Configuration Rename**: Changed `vllm_args` → `vllm_extra_args` for clarity +- **Conditional Behavior**: Different VLLM arg handling based on `use_kserve_defaults` flag + - **KServe Mode**: Only extra args when `use_kserve_defaults: true` + - **Standard Mode**: Extra args + automatic `--tensor-parallel-size` when `use_kserve_defaults: false` +- **P/D Integration**: P/D deployments receive both profile args and global P/D args with correct tensor parallelism + +### Resource Allocation Improvements +- **Per-Pod-Type Configuration**: Different resource allocation for prefill vs decode pods + - **Prefill Pods**: Use prefill tensor parallelism (extracted from `p.tp{N}`) + - **Decode Pods**: Use main tensor parallelism (extracted from `d.tp{N}` or `tp{N}`) +- **Resource Merging**: Automatic merging of base resources with P/D-specific resources + +### Testing Infrastructure Enhancements +- **Annotation-Based Profile Detection**: Added `forge.openshift.io/deployment-profile` annotation for reliable profile identification + - **Replaced Heuristics**: Eliminated complex directory name parsing in test system + - **Reliable Testing**: Tests now read profile names directly from manifest annotations +- **Namespace Configuration**: Fixed namespace override behavior for consistent testing + +### Environment and Artifact Management +- **Artifact Directory Naming**: Enhanced artifact directory creation with conflict resolution + - **Base Format**: `forge_YYYYMMDD-HHMM` without seconds when possible + - **Conflict Resolution**: Add seconds (`forge_YYYYMMDD-HHMM-SS`) when directory exists +- **Symlink Management**: Automatic `/tmp/forge_last` symlink to latest artifact directory + +### Files Modified +- `projects/llm_d/orchestration/runtime_config.py` - FROM_NAME extraction system, template resolution, parameter parsing +- `projects/llm_d/orchestration/render_inference_service.py` - P/D rendering, VLLM args system, resource allocation, annotations +- `projects/llm_d/orchestration/config.d/deployments.yaml` - Template profiles, global P/D configuration +- `projects/llm_d/orchestration/config.d/runtime.yaml` - Namespace override, kueue configuration updates +- `projects/llm_d/tests/test_deployment_profiles.py` - Annotation-based profile detection, namespace override +- `projects/core/library/env.py` - Artifact directory conflict resolution, symlink management +- `projects/llm_d/docs/PD_TESTING.md` - Updated documentation for new P/D configuration system + +### Benefits +- **Flexible Configuration**: Profile names encode all P/D parameters for easy testing +- **Template Reuse**: Single template profile supports multiple P/D configurations +- **Resource Optimization**: Different GPU allocation for prefill vs decode workloads +- **Simplified Testing**: Annotation-based profile detection eliminates complex heuristics +- **Clean Architecture**: Global P/D configuration reduces duplication across profiles + ## 2026-07-16 - Preset Configuration Support ### Configuration Management From a6099dfc8babf5caa2f2812a0d022be0c3a9f298 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:45:56 +0200 Subject: [PATCH 0180/1072] [core] library: postprocess: return the status dict if postprocessing is disabled --- projects/core/library/postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index fa2c8ddb0..5fe2c100a 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -309,7 +309,7 @@ def run_postprocess_after_test( if not postprocess_config.enabled: logger.info("Caliper post-processing disabled (caliper.postprocess.enabled: false).") - return + return dict(success=True, final_status="Post-processing disabled") artifact_root_path = Path(artifact_root).resolve() if artifact_root is not None else None From 103e33cb00510ad2b4e337c3f0c5451554a40423 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:46:41 +0200 Subject: [PATCH 0181/1072] [llm_d] orchestration: use the config file to define the labels --- projects/llm_d/orchestration/config.d/platform.yaml | 4 ++++ projects/llm_d/orchestration/test_phase.py | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 8f592e46f..a8bf91b73 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -5,6 +5,10 @@ cluster: - feature.node.kubernetes.io/pci-10de.present - feature.node.kubernetes.io/pci-0302_10de.present - feature.node.kubernetes.io/pci-0300_10de.present + namespace: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d" operators: openshift-cert-manager-operator: diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 0fa44b436..9e22bc7c8 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -116,11 +116,7 @@ def do_test() -> int: if not dry_run: # Ensure namespace exists before starting any deployments ensure_namespace( - namespace, - labels={ - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - }, + namespace, labels=config.project.get_config("platform.cluster.namespace.labels") ) endpoint_url: str | None = None From bdf4935c4ee7337b456c8a4f1acd863056f324a4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:53:24 +0200 Subject: [PATCH 0182/1072] [llm_d] finish removing the managed namespace --- projects/llm_d/orchestration/phase_inputs.py | 11 -- projects/llm_d/orchestration/prepare_phase.py | 1 - projects/llm_d/tests/test_profiles.py | 112 +----------------- 3 files changed, 5 insertions(+), 119 deletions(-) diff --git a/projects/llm_d/orchestration/phase_inputs.py b/projects/llm_d/orchestration/phase_inputs.py index 70f8b4387..2fbcc62ca 100644 --- a/projects/llm_d/orchestration/phase_inputs.py +++ b/projects/llm_d/orchestration/phase_inputs.py @@ -21,7 +21,6 @@ class PrepareModelCacheInputs: artifact_dir: Path preset_name: str namespace: str - namespace_is_managed: bool model_key: str model: dict[str, Any] model_cache: dict[str, Any] @@ -33,7 +32,6 @@ class PrepareInputs: config_dir: Path preset_name: str namespace: str - namespace_is_managed: bool platform: dict[str, Any] model_key: str model: dict[str, Any] @@ -83,7 +81,6 @@ def write_prepare_inputs_from_prepare(inputs: PrepareInputs) -> Path: "config_dir": str(inputs.config_dir), "preset_name": inputs.preset_name, "namespace": inputs.namespace, - "namespace_is_managed": inputs.namespace_is_managed, "platform": inputs.platform, "model_key": inputs.model_key, "model": inputs.model, @@ -125,7 +122,6 @@ def load_prepare_model_cache_inputs(path: str | Path) -> PrepareModelCacheInputs artifact_dir=Path(payload["artifact_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], model_key=payload["model_key"], model=payload["model"], model_cache=payload["model_cache"], @@ -137,7 +133,6 @@ def build_prepare_model_cache_inputs( artifact_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, model_key: str, model: dict[str, Any], model_cache: dict[str, Any], @@ -146,7 +141,6 @@ def build_prepare_model_cache_inputs( artifact_dir=Path(artifact_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, model_key=model_key, model=model, model_cache=model_cache, @@ -160,7 +154,6 @@ def load_prepare_inputs(path: str | Path) -> PrepareInputs: config_dir=Path(payload["config_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], platform=payload["platform"], model_key=payload["model_key"], model=payload["model"], @@ -175,7 +168,6 @@ def build_prepare_inputs( config_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, platform: dict[str, Any], model_key: str, model: dict[str, Any], @@ -187,7 +179,6 @@ def build_prepare_inputs( config_dir=Path(config_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, platform=platform, model_key=model_key, model=model, @@ -274,7 +265,6 @@ def prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> PrepareMod artifact_dir=inputs.artifact_dir, preset_name=inputs.preset_name, namespace=inputs.namespace, - namespace_is_managed=inputs.namespace_is_managed, model_key=inputs.model_key, model=inputs.model, model_cache=inputs.model_cache, @@ -290,7 +280,6 @@ def write_prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> Path "artifact_dir": str(cache_inputs.artifact_dir), "preset_name": cache_inputs.preset_name, "namespace": cache_inputs.namespace, - "namespace_is_managed": cache_inputs.namespace_is_managed, "model_key": cache_inputs.model_key, "model": cache_inputs.model, "model_cache": cache_inputs.model_cache, diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 930f35e6a..533596f69 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -283,7 +283,6 @@ def prepare_model_cache() -> None: common_args = { "namespace": namespace, - "namespace_is_managed": runtime_config.get_namespace_is_managed(), "model_key": model_slug, "model_uri": model_uri, "pvc_size": model_cache["pvc"]["size"], diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index c4df70be7..54180e8dc 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -58,13 +58,6 @@ def test_release_deployment_profiles_have_expected_shape() -> None: for profile in (approximate, precise, distributed): assert profile["replicas"] == 1 assert profile["tensor_parallelism"] == 1 - assert profile["vllm_args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - ] assert isinstance(approximate["scheduler"], dict) assert isinstance(precise["scheduler"], dict) @@ -188,9 +181,6 @@ def test_release_preset_produces_3_run_specs() -> None: assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) - namespaces = [spec.namespace for spec in run_specs] - assert len(namespaces) == len(set(namespaces)) - def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: variable_overrides_path = env.ARTIFACT_DIR / "000__ci_metadata" / "variable_overrides.yaml" @@ -213,15 +203,6 @@ def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> assert runtime_config.get_benchmark_keys() == ["multi-turn"] -def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided() -> None: - llmd_ci.init() - - assert runtime_config.get_model_name() == "Qwen/Qwen3-0.6B" - assert runtime_config.get_deployment_profile_name() == "approximate-prefix-cache" - # Default preset "smoke" enables the short benchmark - assert runtime_config.get_benchmark_keys() == ["short"] - - def test_list_vaults_only_includes_rhoai_custom_catalog_vaults_for_custom_catalog_runs() -> None: _init_project_config() @@ -574,15 +555,15 @@ def test_3d_matrix_with_multiple_benchmark_keys() -> None: core_config.project.set_config( "runtime.model_name", - "[openai/gpt-oss-120b, Qwen/Qwen3-0.6B]", + ["openai/gpt-oss-120b", "Qwen/Qwen3-0.6B"], ) core_config.project.set_config( "runtime.deployment_profile", - "[distributed-default, precise-prefix-cache]", + ["distributed-default", "precise-prefix-cache"], ) core_config.project.set_config( "runtime.benchmark_key", - "[concurrent-1k-1k, multi-turn]", + ["concurrent-1k-1k", "multi-turn"], ) run_specs = runtime_config.get_run_specs() @@ -596,14 +577,6 @@ def test_3d_matrix_with_multiple_benchmark_keys() -> None: assert run_specs[0].deployment_profile_name == "distributed-default" assert run_specs[0].benchmark_key == "concurrent-1k-1k" - # Verify all namespaces are unique - namespaces = [spec.namespace for spec in run_specs] - assert len(namespaces) == len(set(namespaces)) - - # Verify all artifact dirnames are unique - dirnames = [spec.artifact_dirname for spec in run_specs] - assert len(dirnames) == len(set(dirnames)) - def test_null_benchmark_key_produces_smoke_only_specs() -> None: _init_project_config() @@ -627,7 +600,7 @@ def test_single_benchmark_key_backward_compatible() -> None: assert len(run_specs) == 1 assert run_specs[0].benchmark_key == "short" assert run_specs[0].benchmark_slug == "short" - assert run_specs[0].artifact_dirname == "llmd_run" + assert run_specs[0].artifact_dirname == "llmd__short" def test_activate_run_spec_sets_benchmark_key() -> None: @@ -708,48 +681,9 @@ def test_render_uses_sanitized_model_name_and_profile_resources() -> None: "requests": {"nvidia.com/gpu": "1"}, "limits": {"nvidia.com/gpu": "1"}, } - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - "--tensor-parallel-size=1", - ] assert manifest["spec"]["router"]["scheduler"] == {} -def test_render_preserves_explicit_tensor_parallel_arg() -> None: - _init_project_config() - core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["explicit-tp"] = { - "replicas": 1, - "tensor_parallelism": 2, - "scheduler": {}, - "vllm_args": [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ], - } - core_config.project.set_config("runtime.model_name", "openai/gpt-oss-120b") - core_config.project.set_config("runtime.deployment_profile", "explicit-tp") - - manifest = render_inference_service_from_parts( - config_dir=str(PROJECT_ORCHESTRATION_DIR), - namespace="forge-llm-d", - inference_service=runtime_config.get_platform_config()["inference_service"], - model_name=runtime_config.get_model_name(), - model_slug=runtime_config.get_model_slug(), - deployment_profile=runtime_config.get_deployment_profile(), - model_cache=runtime_config.get_model_cache_config(), - ) - - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ] - - def test_render_uses_embedded_scheduler_config() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) @@ -774,7 +708,7 @@ def test_render_uses_embedded_scheduler_config() -> None: def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["no-scheduler"] = { + core_config.project.config["deployments"]["profiles"]["no-scheduler"] = { "replicas": 1, "tensor_parallelism": 1, "scheduler": None, @@ -818,42 +752,6 @@ def test_smoke_preset_benchmark_behavior() -> None: assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" -def test_run_spec_preserves_namespace_managed_flag() -> None: - _init_project_config() - - core_config.project.apply_preset("smoke") - base_managed = runtime_config.get_namespace_is_managed() - - run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed == base_managed - - with runtime_config.activate_run_spec(run_spec): - # Inside the active spec the override is applied, but managed-ness must - # still reflect the base value, not flip to False. - assert runtime_config.get_namespace_is_managed() == base_managed - # And it is restored on exit. - assert runtime_config.get_namespace_is_managed() == base_managed - - -def test_run_spec_preserves_managed_flag_when_namespace_is_auto_derived() -> None: - """Verify managed flag stays True even when activate_run_spec sets namespace_override.""" - _init_project_config() - - # Force auto-derived namespace by clearing the hardcoded name - core_config.project.config["platform"]["cluster"]["namespace"]["name"] = None - - core_config.project.apply_preset("smoke") - assert runtime_config.get_namespace_is_managed() is True - - run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed is True - - with runtime_config.activate_run_spec(run_spec): - # activate_run_spec sets namespace_override, but managed flag must stay True - assert runtime_config.get_namespace_is_managed() is True - assert runtime_config.get_namespace_is_managed() is True - - @pytest.mark.parametrize( "raw,expected", [ From 20f1854dbc07530c1cb653157fbddc8264e94b5b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:07:41 +0200 Subject: [PATCH 0183/1072] [kserve] toolbox: prepare_hf_model_cache: remove the useless namespace_is_managed flag --- projects/kserve/toolbox/prepare_hf_model_cache/main.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index 572ec43dc..a25544e87 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -35,7 +35,6 @@ def run( *, namespace: str, - namespace_is_managed: bool, model_key: str, model_uri: str, pvc_size: str, @@ -54,7 +53,6 @@ def run( Args: namespace: Namespace where the workload runs - namespace_is_managed: Whether namespace lifecycle is managed by llm_d model_key: Selected model key model_uri: HuggingFace model URI (hf://...) pvc_size: Size of the PVC to create @@ -112,13 +110,6 @@ def build_cache_spec(args, ctx): "oci_registry_auth_secret_key": None, } - if args.namespace_is_managed: - logger.warning( - "Model cache PVC %s lives in managed namespace %s. Namespace cleanup will remove it; cache reuse requires a stable namespace override.", - cache_spec["pvc_name"], - cache_spec["namespace"], - ) - ctx.cache_spec = cache_spec ctx.hf_secret_created = False ctx.hf_secret_name = None From e133a3a122842e44ea57c4475abbdddd7dc2f25c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:17:21 +0200 Subject: [PATCH 0184/1072] [llm_d] relocatethe runtime.kserve.dry_run flag --- projects/llm_d/orchestration/config.d/runtime.yaml | 3 ++- projects/llm_d/orchestration/test_phase.py | 8 ++++---- projects/llm_d/tests/test_deployment_profiles.py | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index c2f2dd14e..81abc4371 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -5,7 +5,8 @@ smoke_request_key: default benchmark_key: null job_name: null namespace: forge-llm-d -kserve_dry_run: false +kserve: + dry_run: false kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 9e22bc7c8..8402815d8 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -50,7 +50,7 @@ def create_test_labels() -> None: def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if dry_run: ret = do_test() logger.info("Kserve dry-run mode enabled - Skipping caliper post-processing") @@ -111,7 +111,7 @@ def do_test() -> int: # Load minimal config needed for orchestration flow namespace = runtime_config.get_namespace() - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if not dry_run: # Ensure namespace exists before starting any deployments @@ -181,7 +181,7 @@ def deploy_inference_service() -> str: gateway = platform["gateway"] # Check if dry-run mode is enabled early - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: @@ -339,7 +339,7 @@ def cleanup_test_resources() -> None: """Cleanup test resources using the toolbox script""" # Skip cleanup when in dry-run mode - dry_run = config.project.get_config("runtime.kserve_dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run", False) if dry_run: logger.info("Skipping cleanup_test_resources - dry-run mode enabled") return diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 9c99c8d52..631ab48a0 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -52,8 +52,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): variable_overrides_path.write_text( yaml.safe_dump( { - "runtime.kserve_dry_run": True, - "runtime.namespace_override": "forge-llm-d", + "runtime.kserve.dry_run": True, }, sort_keys=True, ), From d52fc2534b4fb57dbe6dc0257a4fcf3b8eec72c1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 16:12:33 -0400 Subject: [PATCH 0185/1072] Fix result extraction truncation and PVC cleanup for parallel jobs - Switch GuideLLM result extraction from oc exec cat to oc cp to avoid truncation of large JSON files (~73MB+) streamed through K8s API - Scope post-cleanup to current job's FJOB suffix for parallel safety - Add PVC cleanup alongside job/pod/isvc cleanup - Remove unused webhook functions from core notifications Co-authored-by: Cursor --- projects/core/notifications/send.py | 30 ------------------- .../toolbox/run_guidellm_benchmark/main.py | 16 +++++----- .../rhaiis/orchestration/prepare_rhaiis.py | 27 ++++++++++++++--- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index dfc494765..8f2d55b2e 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -16,7 +16,6 @@ GITHUB_APP_PEM_FILE = "topsail-bot.2024-09-18.private-key.pem" GITHUB_APP_CLIENT_ID_FILE = "topsail-bot.clientid" SLACK_TOKEN_FILE = "topsail-bot.slack-token" -SLACK_WEBHOOK_URL_FILE = "slack-webhook-url" DEFAULT_REPO_OWNER = "openshift-psap" DEFAULT_REPO_NAME = "forge" @@ -715,35 +714,6 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok -def _send_via_webhook(webhook_url: str, blocks: list[dict]) -> bool: - """Post a Slack message via Incoming Webhook URL.""" - from urllib.error import URLError - from urllib.request import Request, urlopen - - body = json.dumps({"blocks": blocks}) - try: - req = Request( - webhook_url, - data=body.encode("utf-8"), - headers={"Content-Type": "application/json"}, - ) - urlopen(req, timeout=15) # noqa: S310 - return True - except (URLError, OSError) as e: - logger.warning("Slack webhook POST failed: %s", e) - return False - - -def _get_webhook_url(vault_name: str) -> str | None: - """Read the webhook URL from a vault file.""" - from projects.core.library import vault as vault_lib - - path = vault_lib.get_vault_content_path(vault_name, SLACK_WEBHOOK_URL_FILE) - if path is None or not path.exists(): - logger.warning("Webhook URL file not found in vault %s", vault_name) - return None - return path.read_text().strip() - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 1d301988b..7095eb7db 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -291,6 +291,8 @@ def extract_results(args, ctx): """Extract GuideLLM results from copy pod""" results_dir = args.artifact_dir / "artifacts" / "results" + results_dir.mkdir(parents=True, exist_ok=True) + copy_pod = f"{ctx.benchmark_name}-copy" extracted_files: list[dict[str, str | None]] = [] for run in ctx.guidellm_runs: if run.rate is None: @@ -301,20 +303,16 @@ def extract_results(args, ctx): local_path = results_dir / f"benchmarks-{run.label}.json" result = oc( - "exec", - "-n", - ctx.target_namespace, - f"{ctx.benchmark_name}-copy", - "--", - "cat", - remote_path, + "cp", + f"{ctx.target_namespace}/{copy_pod}:{remote_path}", + str(local_path), + "-c", "copy-helper", check=False, log_stdout=False, ) - if result.returncode != 0 or not result.stdout: + if result.returncode != 0 or not local_path.exists(): raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") - write_text(local_path, result.stdout) extracted_files.append( { "label": run.label, diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index 5f7421035..29d111ff7 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -1,4 +1,5 @@ import logging +import os from projects.core.dsl.utils.k8s import oc from projects.rhaiis.orchestration import runtime_config @@ -44,12 +45,30 @@ def prepare(): ) +def _delete_resources_by_suffix(resource_type: str, ns: str, suffix: str) -> None: + """Delete resources whose names contain the FJOB suffix.""" + result = oc("get", resource_type, "-n", ns, "-o", "name", check=False, log_stdout=False) + if result.returncode != 0 or not result.stdout: + return + for line in result.stdout.strip().splitlines(): + name = line.strip() + if suffix in name: + oc("delete", name, "-n", ns, "--ignore-not-found", check=False) + + def cleanup(): ns = runtime_config.get_namespace() logger.info(f"Cleaning up rhaiis benchmark resources in {ns}") - oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + fjob = os.environ.get("FJOB_NAME", "") + if fjob: + suffix = fjob.rsplit("-", 1)[-1] + for resource_type in ("inferenceservice", "servingruntime", "job", "pod", "pvc"): + _delete_resources_by_suffix(resource_type, ns, suffix) + else: + oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + logger.info("Cleanup complete") From fdf73c3e244b19b82f3b9141f217674d398960bf Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:15:48 -0400 Subject: [PATCH 0186/1072] Refactor dashboard CSV to use Caliper postprocessing pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace manual benchmarks.json lookup and PSAP payload generation with Caliper plugin's parse → KPI → CSV export workflow - Implement export_kpis_to_csv in RhaiisPlugin to produce dashboard CSV format by pivoting per-rate-point KPIs - Modify RhaiisKpiHandler to emit per-rate-point KPIs from performance_curves instead of aggregate scalars - Enrich test labels with hf_model_id, version, image_tag, cluster_tag, and runtime_args for CSV metadata - Enable kpis_to_csv step in caliper postprocess config - Add _sync_postprocessed_dashboard_csv to sync the generated CSV to S3 and run regression after Caliper postprocessing - Remove _generate_psap_payload and _generate_and_sync_dashboard_csv from per-workload loop - Remove unused generate_psap_payload/write_psap_payload from parser Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.yaml | 5 +- .../rhaiis/orchestration/notifications.py | 39 ++ projects/rhaiis/orchestration/test_phase.py | 187 +++++----- projects/rhaiis/postprocess/kpis.py | 340 +++++++++--------- projects/rhaiis/postprocess/parser.py | 74 ---- projects/rhaiis/postprocess/plugin.py | 89 +++++ 6 files changed, 375 insertions(+), 359 deletions(-) diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 9c6bf6f99..c158eb8eb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -43,8 +43,11 @@ caliper: artifacts_to_kpis: enabled: true output: kpis.json + kpis_to_csv: + enabled: true + output: dashboard.csv csv_dashboard: - enabled: false + enabled: true s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv analyze: enabled: false diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index 486c6c340..98b35a931 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -47,3 +47,42 @@ def send_pipeline_failure_alert( ) except Exception: logger.warning("Failed to send pipeline failure alert", exc_info=True) + + +def send_pipeline_warning( + *, + warnings: list[str], + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert for non-critical pipeline warnings.""" + try: + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + warning_text = "\n".join(f"• {w}" for w in warnings) + send_failure_notification( + error=f"Pipeline completed with warnings:\n{warning_text}", + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to send pipeline warning alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 0cad2b784..f8a6dd487 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) _K8S_NAME_MAX = 63 +_warnings: list[str] = [] @@ -29,7 +30,7 @@ def run( namespace: str, deployment_name: str | None = None, ) -> int: - return run_and_postprocess( + ret = run_and_postprocess( do_test, model_key=model_key, workload_keys=workload_keys, @@ -37,6 +38,13 @@ def run( deployment_name=deployment_name, ) + try: + _sync_postprocessed_dashboard_csv(model_key, workload_keys) + except Exception: + logger.warning("Dashboard CSV S3 sync after postprocessing failed", exc_info=True) + + return ret + def do_test( *, @@ -94,7 +102,16 @@ def _run_test( vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) - _create_test_labels(model_key, workload_keys[0], accelerator, vllm_args) + from projects.core.library import config as _cfg + version = _cfg.project.get_config("tests.rhaiis.version", "") + + _create_test_labels( + model_key, workload_keys[0], accelerator, vllm_args, + hf_model_id=model_cfg["hf_model_id"], + version=version, + vllm_image=vllm_image, + cluster_tag=cluster_tag, + ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, @@ -102,7 +119,6 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - from projects.core.library import config as _cfg run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) @@ -137,8 +153,9 @@ def _run_test( run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) - return 0 + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append("Standalone analysis failed") + return 1 if _warnings else 0 benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) @@ -265,7 +282,19 @@ def _run_test( try: _upload_predictor_log(run_uuid) except Exception: - logger.warning("Predictor log upload failed; continuing", exc_info=True) + logger.warning("Predictor log upload failed", exc_info=True) + _warnings.append("Predictor log upload failed") + + if _warnings: + logger.warning("Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings)) + from projects.rhaiis.orchestration.notifications import send_pipeline_warning + send_pipeline_warning( + warnings=list(_warnings), + model_key=model_key, + workload_keys=workload_keys, + ) + _warnings.clear() + return 1 return 0 @@ -301,14 +330,14 @@ def _run_workload_benchmark( run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - main_benchmark_dir = None if not run_benchmark: logger.info("run_benchmark=false, skipping main benchmark") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: - logger.warning("Standalone analysis failed; continuing", exc_info=True) + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append(f"Standalone analysis failed for {workload_key}") else: logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) @@ -322,7 +351,6 @@ def _run_workload_benchmark( max_seconds=max_seconds, ) - pre_index = env.next_artifact_index() run_guidellm_benchmark( endpoint_url=f"{endpoint_url}/v1", name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), @@ -333,34 +361,34 @@ def _run_workload_benchmark( guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), ) - from pathlib import Path - candidates = sorted(Path(env.ARTIFACT_DIR).glob(f"{pre_index:03d}__*")) - if candidates: - main_benchmark_dir = candidates[0] - if main_benchmark_dir: - try: - _generate_psap_payload( - model_cfg, accelerator_key, vllm_image, vllm_args, workload_key, - run_uuid=run_uuid, benchmark_dir=main_benchmark_dir, - ) - except Exception: - logger.warning("PSAP payload generation failed; continuing", exc_info=True) - - try: - _generate_and_sync_dashboard_csv(model_cfg, accelerator_key, workload_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Dashboard CSV generation/sync failed; continuing", exc_info=True) + # Dashboard CSV generation and S3 sync is handled by the Caliper + # postprocessing pipeline (kpis_to_csv step) + _sync_postprocessed_dashboard_csv def _create_test_labels( - model_key: str, workload_key: str, accelerator: str, vllm_args: dict + model_key: str, + workload_key: str, + accelerator: str, + vllm_args: dict, + *, + hf_model_id: str = "", + version: str = "", + vllm_image: str = "", + cluster_tag: str = "", ) -> None: + _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") + runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) labels = { "model_key": model_key, "workload_key": workload_key, "accelerator": accelerator, "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), + "hf_model_id": hf_model_id, + "version": version, + "image_tag": image_tag, + "cluster_tag": cluster_tag, + "runtime_args": runtime_args, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) @@ -407,92 +435,30 @@ def _set_mlflow_metadata( logger.info("Set MLflow tags: %s", list(tags.keys())) -def _generate_psap_payload( - model_cfg: dict, - accelerator: str, - vllm_image: str, - vllm_args: dict, - workload_key: str, - *, - run_uuid: str = "", - benchmark_dir: Path | None = None, -) -> None: - from pathlib import Path - - from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload - - if benchmark_dir: - benchmarks_json = benchmark_dir / "artifacts" / "results" / "benchmarks.json" - else: - matches = list( - Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") - ) - benchmarks_json = matches[-1] if matches else None - - if not benchmarks_json or not benchmarks_json.exists(): - logger.warning( - "benchmarks.json not found, skipping PSAP payload" - ) - return - - payload = generate_psap_payload( - benchmarks_json_path=benchmarks_json, - model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, - vllm_args=vllm_args, - accelerator=accelerator, - workload_key=workload_key, - run_uuid=run_uuid, - ) - output_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "results" - write_psap_payload( - payload=payload, - output_dir=output_dir, - accelerator=accelerator, - model_id=model_cfg["hf_model_id"], - workload_key=workload_key, - ) - - -def _generate_and_sync_dashboard_csv( - model_cfg: dict, - accelerator_key: str, - workload_key: str, - vllm_args: dict, - *, - run_uuid: str = "", -) -> None: +def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) -> None: + """Find the dashboard CSV generated by Caliper kpis_to_csv and sync to S3.""" from pathlib import Path from projects.core.library import config - from projects.rhaiis.postprocess.csv_export import find_psap_files, generate_dashboard_csv - # Use chip family only (e.g. "H200") — cluster_tag is only for run ID in the CSV - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() - - psap_files = find_psap_files(Path(env.ARTIFACT_DIR)) - if not psap_files: - logger.warning("No PSAP JSON files found, skipping dashboard CSV") + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): + logger.info("Dashboard CSV S3 sync not enabled") return - version = config.project.get_config("tests.rhaiis.version", "") - if not version: - logger.info("No version configured, skipping dashboard CSV generation") + # Find the CSV produced by the kpis_to_csv step in the postprocessing directory + postprocess_dirs = sorted(Path(env.ARTIFACT_DIR).glob("*__postprocessing")) + if not postprocess_dirs: + logger.info("No postprocessing directory found, skipping dashboard CSV sync") return - csv_path = None - for psap_file in psap_files: - output_path = psap_file.parent / f"dashboard_{psap_file.stem}.csv" - csv_path = generate_dashboard_csv(psap_file, version=version, output_path=output_path, run_uuid=run_uuid) - logger.info("Generated dashboard CSV: %s", csv_path) - - if not csv_path: + kpis_to_csv_output = config.project.get_config("caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv") + csv_path = postprocess_dirs[-1] / kpis_to_csv_output + if not csv_path.exists(): + logger.info("Dashboard CSV not found at %s, skipping sync", csv_path) return - csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) - if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): - logger.info("Dashboard CSV S3 sync not enabled") - return + logger.info("Found postprocessed dashboard CSV: %s", csv_path) from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 @@ -507,13 +473,22 @@ def _generate_and_sync_dashboard_csv( ) logger.info("Dashboard CSV sync result: %s", sync_result) + version = config.project.get_config("tests.rhaiis.version", "") compare_version = config.project.get_config("tests.rhaiis.compare_version", "") - if not compare_version: + if not compare_version or not version: return - from projects.rhaiis.orchestration.analysis import run_regression_check - run_regression_check(csv_path, compare_version, version, model_cfg, accelerator, run_uuid=run_uuid, vllm_args=vllm_args) + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + vllm_defaults = runtime_config.get_vllm_defaults() + first_workload = runtime_config.get_workload(workload_keys[0]) + vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check( + csv_path, compare_version, version, model_cfg, accelerator, + run_uuid="", vllm_args=vllm_args, + ) def _upload_predictor_log(run_uuid: str) -> None: diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 7704bd50c..426432e8b 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -10,138 +10,25 @@ class RhaiisKpiHandler: @staticmethod def get_catalog() -> list[dict[str, Any]]: return [ - { - "kpi_id": "guidellm_request_rate", - "name": "Request Rate", - "unit": "req/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_tokens_per_second", - "name": "Total Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_input_tokens_per_second", - "name": "Input Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_output_tokens_per_second", - "name": "Output Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_ttft_median", - "name": "Time to First Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p95", - "name": "Time to First Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p99", - "name": "Time to First Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_median", - "name": "Inter Token Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p95", - "name": "Inter Token Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p99", - "name": "Inter Token Latency (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_median", - "name": "Time Per Output Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p95", - "name": "Time Per Output Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p99", - "name": "Time Per Output Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_median", - "name": "End-to-End Request Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_p95", - "name": "End-to-End Request Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_input_tokens_per_request", - "name": "Input Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_output_tokens_per_request", - "name": "Output Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_concurrency", - "name": "Measured Request Concurrency", - "unit": "count", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_completed_requests", - "name": "Completed Requests", - "unit": "count", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_failed_requests", - "name": "Failed Requests", - "unit": "count", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_duration", - "name": "Benchmark Duration", - "unit": "s", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_prompt_token_count_mean", - "name": "Prompt Token Count (Mean)", - "unit": "tokens", - "higher_is_better": None, - }, + {"kpi_id": "rhaiis_output_tok_per_sec", "name": "Output Token Throughput", "unit": "tokens/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_total_tok_per_sec", "name": "Total Token Throughput", "unit": "tokens/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_measured_rps", "name": "Measured Request Rate", "unit": "req/s", "higher_is_better": True}, + {"kpi_id": "rhaiis_measured_concurrency", "name": "Measured Concurrency", "unit": "count", "higher_is_better": None}, + {"kpi_id": "rhaiis_completed_requests", "name": "Completed Requests", "unit": "count", "higher_is_better": True}, + {"kpi_id": "rhaiis_failed_requests", "name": "Failed Requests", "unit": "count", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_median", "name": "TTFT (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_p95", "name": "TTFT (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_ttft_p99", "name": "TTFT (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_median", "name": "TPOT (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_p95", "name": "TPOT (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_tpot_p99", "name": "TPOT (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_median", "name": "ITL (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_p95", "name": "ITL (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_itl_p99", "name": "ITL (P99)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_request_latency_median", "name": "Request Latency (Median)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_request_latency_p95", "name": "Request Latency (P95)", "unit": "s", "higher_is_better": False}, + {"kpi_id": "rhaiis_prompt_token_count_mean", "name": "Prompt Token Count (Mean)", "unit": "tokens", "higher_is_better": None}, + {"kpi_id": "rhaiis_duration", "name": "Benchmark Duration", "unit": "s", "higher_is_better": None}, ] @staticmethod @@ -149,6 +36,26 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, Any]] = [] + # Mapping from performance_curves keys to KPI IDs + curve_kpi_mappings = [ + ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_completed_requests", "completed_requests", "count", True), + ("rhaiis_failed_requests", "failed_requests", "count", False), + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), + ] + for r in model.unified_result_records: if not r.run_identity.get("guidellm"): continue @@ -156,54 +63,39 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: continue base_labels = {**r.distinguishing_labels} + curves = r.metrics.get("performance_curves", {}) + request_rates = r.metrics.get("request_rate", []) - kpi_mappings = [ - ("guidellm_request_rate", "request_rate", "req/s", True), - ("guidellm_tokens_per_second", "tokens_per_second", "tokens/s", True), - ("guidellm_input_tokens_per_second", "input_tokens_per_second", "tokens/s", True), - ("guidellm_output_tokens_per_second", "output_tokens_per_second", "tokens/s", True), - ("guidellm_ttft_median", "ttft_median", "s", False), - ("guidellm_ttft_p95", "ttft_p95", "s", False), - ("guidellm_ttft_p99", "ttft_p99", "s", False), - ("guidellm_itl_median", "itl_median", "s", False), - ("guidellm_itl_p95", "itl_p95", "s", False), - ("guidellm_itl_p99", "itl_p99", "s", False), - ("guidellm_tpot_median", "tpot_median", "s", False), - ("guidellm_tpot_p95", "tpot_p95", "s", False), - ("guidellm_tpot_p99", "tpot_p99", "s", False), - ("guidellm_request_latency_median", "request_latency_median", "s", False), - ("guidellm_request_latency_p95", "request_latency_p95", "s", False), - ("guidellm_input_tokens_per_request", "input_tokens_per_request", "tokens", False), - ( - "guidellm_output_tokens_per_request", - "output_tokens_per_request", - "tokens", - False, - ), - ("guidellm_request_concurrency", "request_concurrency", "count", None), - ("guidellm_completed_requests", "completed_requests", "count", True), - ("guidellm_failed_requests", "failed_requests", "count", False), - ("guidellm_duration", "duration", "s", None), - ("guidellm_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ] - - for kpi_id, metric_key, unit, higher_is_better in kpi_mappings: - raw_value = r.metrics.get(metric_key) - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None + if not curves or not request_rates: + # Fallback: emit scalar KPIs from top-level metrics + _emit_scalar_kpis(out, r, base_labels, ts, model.plugin_module) + continue - # Skip KPIs with null values - if value is None: - continue + # Emit per-rate-point KPIs from performance curves + for idx, rate in enumerate(request_rates): + rate_labels = { + **base_labels, + "rate_index": str(idx), + "intended_concurrency": str(rate), + } - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better + for kpi_id, curve_key, unit, higher_is_better in curve_kpi_mappings: + curve_values = curves.get(curve_key, []) + if idx >= len(curve_values): + continue + raw_value = curve_values[idx] + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**rate_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better - out.append( - { + out.append({ "schema_version": "1", "kpi_id": kpi_id, "value": value, @@ -215,7 +107,99 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: "test_base_path": r.test_base_path, "plugin_module": model.plugin_module, }, - } - ) + }) + + # Also emit scalar KPIs that don't come from curves + for kpi_id, metric_key, unit, higher_is_better in [ + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_duration", "duration", "s", None), + ("rhaiis_measured_rps", "request_rate", "req/s", True), + ]: + raw_value = r.metrics.get(metric_key) + # request_rate is a list; skip it for scalar emission + if isinstance(raw_value, list): + continue + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**base_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better + + out.append({ + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": model.plugin_module, + }, + }) return out + + +def _emit_scalar_kpis( + out: list[dict[str, Any]], + r, + base_labels: dict[str, Any], + ts: str, + plugin_module: str, +) -> None: + """Fallback: emit KPIs from top-level scalar metrics when no curves are available.""" + scalar_mappings = [ + ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_completed_requests", "completed_requests", "count", True), + ("rhaiis_failed_requests", "failed_requests", "count", False), + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_duration", "duration", "s", None), + ] + for kpi_id, metric_key, unit, higher_is_better in scalar_mappings: + raw_value = r.metrics.get(metric_key) + if isinstance(raw_value, list): + continue + try: + value = float(raw_value) if raw_value is not None else None + except (TypeError, ValueError): + value = None + if value is None: + continue + + labels = {**base_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better + + out.append({ + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": plugin_module, + }, + }) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index c0151a298..92b971442 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -2,9 +2,6 @@ import json import logging -import uuid -from datetime import UTC, datetime -from pathlib import Path from typing import Any from projects.caliper.engine.model import ( @@ -109,74 +106,3 @@ def _stat(metric_name: str, stat: str, default: float = 0.0) -> float: return extra -def generate_psap_payload( - *, - benchmarks_json_path: Path, - model_id: str, - vllm_image: str, - vllm_args: dict[str, Any], - accelerator: str, - workload_key: str, - run_uuid: str = "", -) -> dict[str, Any]: - report = json.loads(benchmarks_json_path.read_text(encoding="utf-8")) - - image, tag = _split_image_tag(vllm_image) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) - - guidellm_start = None - guidellm_end = None - benchmarks = report.get("benchmarks", []) - if benchmarks: - starts = [b.get("start_time", 0) for b in benchmarks if b.get("start_time")] - ends = [b.get("end_time", 0) for b in benchmarks if b.get("end_time")] - if starts: - guidellm_start = int(min(starts) * 1000) - if ends: - guidellm_end = int(max(ends) * 1000) - - return { - "experiment_id": run_uuid or str(uuid.uuid4()).upper(), - "experiment_type": "perf", - "model": model_id, - "inference_server": "vllm", - "inference_server_version": tag, - "container_image": image, - "container_image_tag": tag, - "container_entrypoint": None, - "inference_server_args": dict(vllm_args), - "accelerator_type": accelerator.upper(), - "accelerator_count": tp_size, - "accelerator_memory_gb": 0, - "machine_type": None, - "provider": "redhat", - "report": report, - "timestamp": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S%z"), - "guidellm_start_time_ms": guidellm_start, - "guidellm_end_time_ms": guidellm_end, - } - - -def write_psap_payload( - *, - payload: dict[str, Any], - output_dir: Path, - accelerator: str, - model_id: str, - workload_key: str, -) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - model_short = model_id.rsplit("/", 1)[-1].lower().replace(".", "-") - date_str = datetime.now(UTC).strftime("%Y%m%d") - filename = f"PSAP_perf_{workload_key}_{accelerator}_{model_short}_{date_str}.json" - path = output_dir / filename - path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - logger.info("PSAP payload written to %s", path) - return path - - -def _split_image_tag(full_image: str) -> tuple[str, str]: - if ":" in full_image: - parts = full_image.rsplit(":", 1) - return parts[0], parts[1] - return full_image, "latest" diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 3b2809d91..60de3a313 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -1,6 +1,8 @@ from __future__ import annotations +import csv import logging +from collections import defaultdict from pathlib import Path from typing import Any @@ -17,6 +19,28 @@ logger = logging.getLogger(__name__) +# KPI ID → dashboard CSV column name mapping +_KPI_TO_CSV_COLUMN = { + "rhaiis_output_tok_per_sec": "output_tok/sec", + "rhaiis_total_tok_per_sec": "total_tok/sec", + "rhaiis_measured_concurrency": "measured concurrency", + "rhaiis_completed_requests": "successful_requests", + "rhaiis_failed_requests": "errored_requests", + "rhaiis_ttft_median": "ttft_median", + "rhaiis_ttft_p95": "ttft_p95", + "rhaiis_ttft_p99": "ttft_p99", + "rhaiis_tpot_median": "tpot_median", + "rhaiis_tpot_p95": "tpot_p95", + "rhaiis_tpot_p99": "tpot_p99", + "rhaiis_itl_median": "itl_median", + "rhaiis_itl_p95": "itl_p95", + "rhaiis_itl_p99": "itl_p99", + "rhaiis_request_latency_median": "request_latency_median", + "rhaiis_request_latency_p95": "request_latency_max", + "rhaiis_prompt_token_count_mean": "prompt_token_count_mean", +} + + class RhaiisPlugin(PostProcessingPlugin): def __init__(self) -> None: self.parser = RhaiisParser() @@ -53,6 +77,71 @@ def kpi_catalog(self) -> list[dict[str, Any]]: def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to dashboard-format CSV. + + Groups per-rate-point KPIs by (run_path, rate_index) and pivots them into + rows matching the RHAIIS dashboard CSV schema. + """ + from projects.rhaiis.postprocess.csv_export import FIELDNAMES + + # Group KPIs by (run_path, rate_index) + groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(dict) + group_labels: dict[tuple[str, str], dict[str, Any]] = {} + + for kpi in kpi_records: + labels = kpi.get("labels", {}) + run_path = kpi.get("run_path", "") + rate_index = labels.get("rate_index", "0") + key = (run_path, rate_index) + + csv_col = _KPI_TO_CSV_COLUMN.get(kpi.get("kpi_id", "")) + if csv_col: + groups[key][csv_col] = kpi.get("value") + + if key not in group_labels: + group_labels[key] = labels + + # Build CSV rows + rows = [] + for key in sorted(groups): + metrics = groups[key] + labels = group_labels.get(key, {}) + + acc = labels.get("accelerator", "") + cluster_tag = labels.get("cluster_tag", "") + model_id = labels.get("hf_model_id", "") + tp = labels.get("tensor_parallel_size", "1") + version = labels.get("version", "") + run_name = f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" + + row = {col: "" for col in FIELDNAMES} + row.update(metrics) + row["run"] = run_name + row["accelerator"] = acc + row["model"] = model_id + row["version"] = version + row["TP"] = tp + row["intended concurrency"] = labels.get("intended_concurrency", "") + row["image_tag"] = labels.get("image_tag", "") + row["runtime_args"] = labels.get("runtime_args", "") + row["uuid"] = labels.get("run_uuid", "") + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + logger.info("Exported %d dashboard CSV rows to %s", len(rows), output_path) + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: return {} From 4d6e58d4339aab900d02b6222725e02c7c8a511c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:25:07 -0400 Subject: [PATCH 0187/1072] Populate all dashboard CSV fields from Caliper KPIs - Extend RHAIIS parser to extract ALL per-benchmark metrics from raw benchmarks.json: ttft/tpot/itl p1/p999/mean, request_latency min/max, measured_rps, token count percentiles, output/total tok/sec, intended_concurrency, successful/errored requests - Add report-level metadata: guidellm_version, prompt_toks, output_toks, guidellm_start/end_time_ms - Store extra per-benchmark metrics as additional performance_curves entries merged into the parsed record - Add comprehensive KPI mappings covering every dashboard CSV column - Update plugin CSV export to populate all metadata fields from KPI labels (version, image_tag, runtime_args, timestamps, etc.) Co-authored-by: Cursor --- projects/rhaiis/postprocess/kpis.py | 204 ++++++++------------------ projects/rhaiis/postprocess/parser.py | 160 ++++++++++++++++---- projects/rhaiis/postprocess/plugin.py | 49 +++++-- 3 files changed, 234 insertions(+), 179 deletions(-) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 426432e8b..6453ee7a8 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -5,30 +5,57 @@ from projects.caliper.engine.model import UnifiedRunModel +# All per-rate-point metrics we emit as KPIs. +# (kpi_id, performance_curves key, unit, higher_is_better) +_CURVE_KPI_MAPPINGS: list[tuple[str, str, str, bool | None]] = [ + # Throughput + ("rhaiis_output_tok_per_sec", "output_tok_per_sec", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "total_tok_per_sec", "tokens/s", True), + # Concurrency / rate + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_measured_rps", "measured_rps", "req/s", True), + ("rhaiis_intended_concurrency", "intended_concurrency", "count", None), + # Request counts + ("rhaiis_completed_requests", "successful_requests", "count", True), + ("rhaiis_failed_requests", "errored_requests", "count", False), + # TTFT + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_ttft_p1", "ttft_p1", "s", False), + ("rhaiis_ttft_p999", "ttft_p999", "s", False), + ("rhaiis_ttft_mean", "ttft_mean", "s", False), + # TPOT + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_tpot_p1", "tpot_p1", "s", False), + ("rhaiis_tpot_p999", "tpot_p999", "s", False), + # ITL + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_itl_p1", "itl_p1", "s", False), + ("rhaiis_itl_p999", "itl_p999", "s", False), + ("rhaiis_itl_mean", "itl_mean", "s", False), + # Request latency + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_min", "request_latency_min", "s", False), + ("rhaiis_request_latency_max", "request_latency_max", "s", False), + # Token counts + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_prompt_token_count_p99", "prompt_token_count_p99", "tokens", None), + ("rhaiis_output_token_count_mean", "output_token_count_mean", "tokens", None), + ("rhaiis_output_token_count_p99", "output_token_count_p99", "tokens", None), +] + class RhaiisKpiHandler: @staticmethod def get_catalog() -> list[dict[str, Any]]: return [ - {"kpi_id": "rhaiis_output_tok_per_sec", "name": "Output Token Throughput", "unit": "tokens/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_total_tok_per_sec", "name": "Total Token Throughput", "unit": "tokens/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_measured_rps", "name": "Measured Request Rate", "unit": "req/s", "higher_is_better": True}, - {"kpi_id": "rhaiis_measured_concurrency", "name": "Measured Concurrency", "unit": "count", "higher_is_better": None}, - {"kpi_id": "rhaiis_completed_requests", "name": "Completed Requests", "unit": "count", "higher_is_better": True}, - {"kpi_id": "rhaiis_failed_requests", "name": "Failed Requests", "unit": "count", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_median", "name": "TTFT (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_p95", "name": "TTFT (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_ttft_p99", "name": "TTFT (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_median", "name": "TPOT (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_p95", "name": "TPOT (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_tpot_p99", "name": "TPOT (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_median", "name": "ITL (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_p95", "name": "ITL (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_itl_p99", "name": "ITL (P99)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_request_latency_median", "name": "Request Latency (Median)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_request_latency_p95", "name": "Request Latency (P95)", "unit": "s", "higher_is_better": False}, - {"kpi_id": "rhaiis_prompt_token_count_mean", "name": "Prompt Token Count (Mean)", "unit": "tokens", "higher_is_better": None}, - {"kpi_id": "rhaiis_duration", "name": "Benchmark Duration", "unit": "s", "higher_is_better": None}, + {"kpi_id": kpi_id, "name": kpi_id, "unit": unit, "higher_is_better": hib} + for kpi_id, _, unit, hib in _CURVE_KPI_MAPPINGS ] @staticmethod @@ -36,26 +63,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, Any]] = [] - # Mapping from performance_curves keys to KPI IDs - curve_kpi_mappings = [ - ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), - ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), - ("rhaiis_measured_concurrency", "request_concurrency", "count", None), - ("rhaiis_completed_requests", "completed_requests", "count", True), - ("rhaiis_failed_requests", "failed_requests", "count", False), - ("rhaiis_ttft_median", "ttft_median", "s", False), - ("rhaiis_ttft_p95", "ttft_p95", "s", False), - ("rhaiis_ttft_p99", "ttft_p99", "s", False), - ("rhaiis_tpot_median", "tpot_median", "s", False), - ("rhaiis_tpot_p95", "tpot_p95", "s", False), - ("rhaiis_tpot_p99", "tpot_p99", "s", False), - ("rhaiis_itl_median", "itl_median", "s", False), - ("rhaiis_itl_p95", "itl_p95", "s", False), - ("rhaiis_itl_p99", "itl_p99", "s", False), - ("rhaiis_request_latency_median", "request_latency_median", "s", False), - ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), - ] - for r in model.unified_result_records: if not r.run_identity.get("guidellm"): continue @@ -66,29 +73,37 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: curves = r.metrics.get("performance_curves", {}) request_rates = r.metrics.get("request_rate", []) + # Report-level metadata — same for every rate point in this record + meta_labels = { + "guidellm_version": str(r.metrics.get("guidellm_version", "")), + "prompt_toks": str(r.metrics.get("prompt_toks", "")), + "output_toks": str(r.metrics.get("output_toks", "")), + "guidellm_start_time_ms": str(r.metrics.get("guidellm_start_time_ms", "")), + "guidellm_end_time_ms": str(r.metrics.get("guidellm_end_time_ms", "")), + } + if not curves or not request_rates: - # Fallback: emit scalar KPIs from top-level metrics - _emit_scalar_kpis(out, r, base_labels, ts, model.plugin_module) continue - # Emit per-rate-point KPIs from performance curves - for idx, rate in enumerate(request_rates): + n_rates = len(request_rates) + + for idx in range(n_rates): rate_labels = { **base_labels, + **meta_labels, "rate_index": str(idx), - "intended_concurrency": str(rate), } - for kpi_id, curve_key, unit, higher_is_better in curve_kpi_mappings: + for kpi_id, curve_key, unit, higher_is_better in _CURVE_KPI_MAPPINGS: curve_values = curves.get(curve_key, []) if idx >= len(curve_values): continue raw_value = curve_values[idx] + if raw_value is None: + continue try: - value = float(raw_value) if raw_value is not None else None + value = float(raw_value) except (TypeError, ValueError): - value = None - if value is None: continue labels = {**rate_labels} @@ -109,97 +124,4 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: }, }) - # Also emit scalar KPIs that don't come from curves - for kpi_id, metric_key, unit, higher_is_better in [ - ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ("rhaiis_duration", "duration", "s", None), - ("rhaiis_measured_rps", "request_rate", "req/s", True), - ]: - raw_value = r.metrics.get(metric_key) - # request_rate is a list; skip it for scalar emission - if isinstance(raw_value, list): - continue - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None - if value is None: - continue - - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better - - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": model.plugin_module, - }, - }) - return out - - -def _emit_scalar_kpis( - out: list[dict[str, Any]], - r, - base_labels: dict[str, Any], - ts: str, - plugin_module: str, -) -> None: - """Fallback: emit KPIs from top-level scalar metrics when no curves are available.""" - scalar_mappings = [ - ("rhaiis_output_tok_per_sec", "output_tokens_per_second", "tokens/s", True), - ("rhaiis_total_tok_per_sec", "tokens_per_second", "tokens/s", True), - ("rhaiis_measured_concurrency", "request_concurrency", "count", None), - ("rhaiis_completed_requests", "completed_requests", "count", True), - ("rhaiis_failed_requests", "failed_requests", "count", False), - ("rhaiis_ttft_median", "ttft_median", "s", False), - ("rhaiis_ttft_p95", "ttft_p95", "s", False), - ("rhaiis_ttft_p99", "ttft_p99", "s", False), - ("rhaiis_tpot_median", "tpot_median", "s", False), - ("rhaiis_tpot_p95", "tpot_p95", "s", False), - ("rhaiis_tpot_p99", "tpot_p99", "s", False), - ("rhaiis_itl_median", "itl_median", "s", False), - ("rhaiis_itl_p95", "itl_p95", "s", False), - ("rhaiis_itl_p99", "itl_p99", "s", False), - ("rhaiis_request_latency_median", "request_latency_median", "s", False), - ("rhaiis_request_latency_p95", "request_latency_p95", "s", False), - ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ("rhaiis_duration", "duration", "s", None), - ] - for kpi_id, metric_key, unit, higher_is_better in scalar_mappings: - raw_value = r.metrics.get(metric_key) - if isinstance(raw_value, list): - continue - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None - if value is None: - continue - - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better - - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": plugin_module, - }, - }) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 92b971442..1e2a3eccd 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -2,6 +2,7 @@ import json import logging +import re from typing import Any from projects.caliper.engine.model import ( @@ -35,6 +36,12 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: if node: extra = _extract_extra_metrics(node) merged_metrics = {**record.metrics, **extra} + # Merge extra curves into existing performance_curves + existing_curves = merged_metrics.get("performance_curves", {}) + extra_curves = extra.pop("_extra_curves", {}) + if extra_curves: + existing_curves.update(extra_curves) + merged_metrics["performance_curves"] = existing_curves enriched_records.append( UnifiedResultRecord( test_base_path=record.test_base_path, @@ -62,47 +69,142 @@ def _find_node_for_record( def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: extra: dict[str, Any] = {} - benchmarks_files = [p for p in node.artifact_paths if p.name == "benchmarks.json"] + benchmarks_files = sorted( + [p for p in node.artifact_paths if p.name == "benchmarks.json" + or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json")], + key=lambda p: p.name, + ) if not benchmarks_files: return extra - try: - data = json.loads(benchmarks_files[0].read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): + # Collect all benchmarks across all files + all_benchmarks: list[dict] = [] + report_metadata: dict[str, Any] = {} + report_args: dict[str, Any] = {} + + for bf in benchmarks_files: + try: + data = json.loads(bf.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + all_benchmarks.extend(data.get("benchmarks", [])) + if not report_metadata: + report_metadata = data.get("metadata", {}) + if not report_args: + report_args = data.get("args", {}) + + if not all_benchmarks: return extra - benchmarks = data.get("benchmarks", []) - if not benchmarks: - return extra - - bench = benchmarks[0] - metrics = bench.get("metrics", {}) - - def _percentile(metric_name: str, pct: str, default: float = 0.0) -> float: + # Extract report-level metadata + extra["guidellm_version"] = report_metadata.get("guidellm_version", "") + + data_str = "" + if isinstance(report_args, dict): + data_list = report_args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + extra["prompt_toks"] = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + extra["output_toks"] = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + # Compute global start/end timestamps + start_times = [] + end_times = [] + for bench in all_benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + extra["guidellm_start_time_ms"] = int(min(start_times) * 1000) if start_times else "" + extra["guidellm_end_time_ms"] = int(max(end_times) * 1000) if end_times else "" + + # Scalar extras from first benchmark (backward compat) + bench0 = all_benchmarks[0] + m0 = bench0.get("metrics", {}) + + def _percentile0(metric_name: str, pct: str, default: float = 0.0) -> float: return float( - metrics.get(metric_name, {}) - .get("successful", {}) - .get("percentiles", {}) - .get(pct, default) + m0.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct, default) ) - def _stat(metric_name: str, stat: str, default: float = 0.0) -> float: - return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, default)) + def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: + return float(m0.get(metric_name, {}).get("successful", {}).get(stat, default)) - extra["ttft_p99"] = _percentile("time_to_first_token_ms", "p99") / 1000.0 - extra["tpot_p99"] = _percentile("time_per_output_token_ms", "p99") / 1000.0 - extra["itl_p99"] = _percentile("inter_token_latency_ms", "p99") / 1000.0 + extra["ttft_p99"] = _percentile0("time_to_first_token_ms", "p99") / 1000.0 + extra["tpot_p99"] = _percentile0("time_per_output_token_ms", "p99") / 1000.0 + extra["itl_p99"] = _percentile0("inter_token_latency_ms", "p99") / 1000.0 - request_totals = metrics.get("request_totals", {}) - extra["completed_requests"] = int(request_totals.get("successful", 0)) - extra["failed_requests"] = int(request_totals.get("errored", 0)) + request_totals0 = m0.get("request_totals", {}) + extra["completed_requests"] = int(request_totals0.get("successful", 0)) + extra["failed_requests"] = int(request_totals0.get("errored", 0)) + extra["prompt_token_count_mean"] = _stat0("prompt_token_count", "mean") - extra["prompt_token_count_mean"] = _stat("prompt_token_count", "mean") - - concurrency = _stat("request_concurrency", "mean") + concurrency = _stat0("request_concurrency", "mean") if concurrency > 0: extra["request_concurrency"] = concurrency + # Build per-benchmark extra curves for ALL dashboard metrics + # These will be merged into the existing performance_curves dict + extra_curves: dict[str, list] = { + "ttft_p1": [], + "ttft_p999": [], + "ttft_mean": [], + "tpot_p1": [], + "tpot_p999": [], + "itl_p1": [], + "itl_p999": [], + "itl_mean": [], + "request_latency_min": [], + "request_latency_max": [], + "measured_rps": [], + "prompt_token_count_mean": [], + "prompt_token_count_p99": [], + "output_token_count_mean": [], + "output_token_count_p99": [], + "output_tok_per_sec": [], + "total_tok_per_sec": [], + "successful_requests": [], + "errored_requests": [], + "intended_concurrency": [], + } + + for bench in all_benchmarks: + metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + + def _pct(metric_name: str, pct: str): + return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + + def _stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("successful", {}).get(stat) + + def _total_stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("total", {}).get(stat) + + request_totals = metrics.get("request_totals", {}) + + extra_curves["ttft_p1"].append(_pct("time_to_first_token_ms", "p01")) + extra_curves["ttft_p999"].append(_pct("time_to_first_token_ms", "p999")) + extra_curves["ttft_mean"].append(_stat("time_to_first_token_ms", "mean")) + extra_curves["tpot_p1"].append(_pct("time_per_output_token_ms", "p01")) + extra_curves["tpot_p999"].append(_pct("time_per_output_token_ms", "p999")) + extra_curves["itl_p1"].append(_pct("inter_token_latency_ms", "p01")) + extra_curves["itl_p999"].append(_pct("inter_token_latency_ms", "p999")) + extra_curves["itl_mean"].append(_stat("inter_token_latency_ms", "mean")) + extra_curves["request_latency_min"].append(_stat("request_latency", "min")) + extra_curves["request_latency_max"].append(_stat("request_latency", "max")) + extra_curves["measured_rps"].append(_stat("requests_per_second", "mean")) + extra_curves["prompt_token_count_mean"].append(_stat("prompt_token_count", "mean")) + extra_curves["prompt_token_count_p99"].append(_pct("prompt_token_count", "p99")) + extra_curves["output_token_count_mean"].append(_stat("output_token_count", "mean")) + extra_curves["output_token_count_p99"].append(_pct("output_token_count", "p99")) + extra_curves["output_tok_per_sec"].append(_total_stat("output_tokens_per_second", "mean")) + extra_curves["total_tok_per_sec"].append(_total_stat("tokens_per_second", "mean")) + extra_curves["successful_requests"].append(request_totals.get("successful", 0)) + extra_curves["errored_requests"].append(request_totals.get("errored", 0)) + extra_curves["intended_concurrency"].append(strategy.get("streams")) + + extra["_extra_curves"] = extra_curves return extra - - diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 60de3a313..4d67db20d 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -19,25 +19,39 @@ logger = logging.getLogger(__name__) -# KPI ID → dashboard CSV column name mapping +# KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { "rhaiis_output_tok_per_sec": "output_tok/sec", "rhaiis_total_tok_per_sec": "total_tok/sec", "rhaiis_measured_concurrency": "measured concurrency", + "rhaiis_measured_rps": "measured rps", + "rhaiis_intended_concurrency": "intended concurrency", "rhaiis_completed_requests": "successful_requests", "rhaiis_failed_requests": "errored_requests", "rhaiis_ttft_median": "ttft_median", "rhaiis_ttft_p95": "ttft_p95", "rhaiis_ttft_p99": "ttft_p99", + "rhaiis_ttft_p1": "ttft_p1", + "rhaiis_ttft_p999": "ttft_p999", + "rhaiis_ttft_mean": "ttft_mean", "rhaiis_tpot_median": "tpot_median", "rhaiis_tpot_p95": "tpot_p95", "rhaiis_tpot_p99": "tpot_p99", + "rhaiis_tpot_p1": "tpot_p1", + "rhaiis_tpot_p999": "tpot_p999", "rhaiis_itl_median": "itl_median", "rhaiis_itl_p95": "itl_p95", "rhaiis_itl_p99": "itl_p99", + "rhaiis_itl_p1": "itl_p1", + "rhaiis_itl_p999": "itl_p999", + "rhaiis_itl_mean": "itl_mean", "rhaiis_request_latency_median": "request_latency_median", - "rhaiis_request_latency_p95": "request_latency_max", + "rhaiis_request_latency_min": "request_latency_min", + "rhaiis_request_latency_max": "request_latency_max", "rhaiis_prompt_token_count_mean": "prompt_token_count_mean", + "rhaiis_prompt_token_count_p99": "prompt_token_count_p99", + "rhaiis_output_token_count_mean": "output_token_count_mean", + "rhaiis_output_token_count_p99": "output_token_count_p99", } @@ -85,8 +99,9 @@ def export_kpis_to_csv( ) -> str: """Export KPI records to dashboard-format CSV. - Groups per-rate-point KPIs by (run_path, rate_index) and pivots them into - rows matching the RHAIIS dashboard CSV schema. + Groups per-rate-point KPIs by (run_path, rate_index) and pivots them + into rows matching the RHAIIS dashboard CSV schema with all fields + populated. """ from projects.rhaiis.postprocess.csv_export import FIELDNAMES @@ -104,7 +119,8 @@ def export_kpis_to_csv( if csv_col: groups[key][csv_col] = kpi.get("value") - if key not in group_labels: + # Capture the richest set of labels for this group + if key not in group_labels or len(labels) > len(group_labels[key]): group_labels[key] = labels # Build CSV rows @@ -113,24 +129,39 @@ def export_kpis_to_csv( metrics = groups[key] labels = group_labels.get(key, {}) - acc = labels.get("accelerator", "") + acc = labels.get("accelerator", "").upper() cluster_tag = labels.get("cluster_tag", "") model_id = labels.get("hf_model_id", "") tp = labels.get("tensor_parallel_size", "1") version = labels.get("version", "") - run_name = f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" - row = {col: "" for col in FIELDNAMES} + run_name = ( + f"{acc}-{cluster_tag}-{model_id}-{tp}" + if cluster_tag + else f"{acc}-{model_id}-{tp}" + ) + + row: dict[str, Any] = {col: "" for col in FIELDNAMES} + # Populate all metric columns from KPI values row.update(metrics) + # Populate metadata columns from labels row["run"] = run_name row["accelerator"] = acc row["model"] = model_id row["version"] = version row["TP"] = tp - row["intended concurrency"] = labels.get("intended_concurrency", "") + row["prompt toks"] = labels.get("prompt_toks", "") + row["output toks"] = labels.get("output_toks", "") + # intended concurrency is populated from the rhaiis_intended_concurrency KPI; + # only fall back to labels if the KPI didn't set it + if not row.get("intended concurrency"): + row["intended concurrency"] = labels.get("intended_concurrency", "") row["image_tag"] = labels.get("image_tag", "") row["runtime_args"] = labels.get("runtime_args", "") row["uuid"] = labels.get("run_uuid", "") + row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") + row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") + row["guidellm_version"] = labels.get("guidellm_version", "") rows.append(row) output_path.parent.mkdir(parents=True, exist_ok=True) From d15938f7a319b970455a86104815ae05a852fbeb Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 17:32:59 -0400 Subject: [PATCH 0188/1072] Handle large benchmark files: compress before oc cp oc cp fails with 'unexpected EOF' on large benchmarks.json files (~70MB+). Fix by compressing the file inside the copy pod with gzip first, copying the compressed version, then decompressing locally. Falls back to direct oc cp for small files that succeed on the first attempt. Co-authored-by: Cursor --- .../toolbox/run_guidellm_benchmark/main.py | 70 ++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 7095eb7db..632c96efb 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -302,15 +302,8 @@ def extract_results(args, ctx): remote_path = f"/results/benchmarks-{run.label}.json" local_path = results_dir / f"benchmarks-{run.label}.json" - result = oc( - "cp", - f"{ctx.target_namespace}/{copy_pod}:{remote_path}", - str(local_path), - "-c", "copy-helper", - check=False, - log_stdout=False, - ) - if result.returncode != 0 or not local_path.exists(): + local_path = _copy_result_file(ctx.target_namespace, copy_pod, remote_path, local_path) + if local_path is None: raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") extracted_files.append( @@ -333,6 +326,65 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" +def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "Path | None": + """Copy a result file from a pod, compressing first to handle large files. + + Falls back to direct ``oc cp`` for small files. + """ + import gzip + import shutil + + # Try direct oc cp first + result = oc( + "cp", + f"{namespace}/{pod}:{remote_path}", + str(local_path), + "-c", "copy-helper", + check=False, + log_stdout=False, + ) + if result.returncode == 0 and local_path.exists() and local_path.stat().st_size > 0: + return local_path + + logging.getLogger(__name__).info( + "Direct oc cp failed (rc=%d), retrying with gzip compression", result.returncode, + ) + + # Compress inside the pod, copy the gz, decompress locally + gz_remote = f"{remote_path}.gz" + compress = oc( + "exec", pod, "-n", namespace, "-c", "copy-helper", "--", + "gzip", "-kf", remote_path, + check=False, + ) + if compress.returncode != 0: + logging.getLogger(__name__).warning("gzip inside pod failed (rc=%d)", compress.returncode) + return None + + local_gz = local_path.parent / f"{local_path.name}.gz" + cp_result = oc( + "cp", + f"{namespace}/{pod}:{gz_remote}", + str(local_gz), + "-c", "copy-helper", + check=False, + log_stdout=False, + ) + if cp_result.returncode != 0 or not local_gz.exists() or local_gz.stat().st_size == 0: + logging.getLogger(__name__).warning("oc cp of gzipped file failed") + local_gz.unlink(missing_ok=True) + return None + + with gzip.open(local_gz, "rb") as f_in, open(local_path, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + local_gz.unlink(missing_ok=True) + + if local_path.exists() and local_path.stat().st_size > 0: + return local_path + + return None + + @task def cleanup_copy_pod(args, ctx): """Delete the copy pod after results extraction""" From 2b55199c45421819c5109ee5f348caa5c4a8b86c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:08:48 -0400 Subject: [PATCH 0189/1072] Fix KPI validation: add required run_id field Co-authored-by: Cursor --- projects/rhaiis/postprocess/kpis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 6453ee7a8..a262bb41d 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -115,6 +115,7 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: "kpi_id": kpi_id, "value": value, "unit": unit, + "run_id": r.test_base_path, "run_path": r.test_base_path, "timestamp": ts, "labels": labels, From af3b991b247d29a82e7f1cb9a5ecfc3c5f919848 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:10:53 -0400 Subject: [PATCH 0190/1072] Fix dashboard CSV: use GPU chip name for accelerator, include run_uuid - Pass accelerator_chip (e.g. H200) to test labels instead of vendor name (nvidia) so dashboard shows correct accelerator - Move run_uuid generation before _create_test_labels so it propagates into KPI records and dashboard CSV uuid column Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f8a6dd487..01793a977 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -104,6 +104,8 @@ def _run_test( from projects.core.library import config as _cfg version = _cfg.project.get_config("tests.rhaiis.version", "") + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) + logger.info("Run UUID for this job: %s", run_uuid) _create_test_labels( model_key, workload_keys[0], accelerator, vllm_args, @@ -111,6 +113,8 @@ def _run_test( version=version, vllm_image=vllm_image, cluster_tag=cluster_tag, + accelerator_chip=gpu_type.upper(), + run_uuid=run_uuid, ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -119,9 +123,6 @@ def _run_test( from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) - logger.info("Run UUID for this job: %s", run_uuid) - import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") @@ -376,19 +377,22 @@ def _create_test_labels( version: str = "", vllm_image: str = "", cluster_tag: str = "", + accelerator_chip: str = "", + run_uuid: str = "", ) -> None: _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) labels = { "model_key": model_key, "workload_key": workload_key, - "accelerator": accelerator, + "accelerator": accelerator_chip or accelerator, "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), "hf_model_id": hf_model_id, "version": version, "image_tag": image_tag, "cluster_tag": cluster_tag, "runtime_args": runtime_args, + "run_uuid": run_uuid, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) From c77bd806d66df780dc5f2ee840df4d5048e33268 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 22 Jul 2026 18:22:26 -0400 Subject: [PATCH 0191/1072] fix: sort extra curves by request_rate and convert latency units to ms - Sort benchmarks in RHAIIS parser by request_rate to match GuideLLM parser ordering, preventing curve index misalignment - Convert latency extra curves (TTFT/TPOT/ITL) from ms to seconds in parser to match GuideLLM base curves - Convert latency values back to ms in CSV export for dashboard compat - Return extra_curves separately from scalar metrics to avoid _extra_curves key leaking into merged_metrics Co-authored-by: Cursor --- projects/rhaiis/postprocess/parser.py | 64 +++++++++++++++++---------- projects/rhaiis/postprocess/plugin.py | 15 +++++++ 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 1e2a3eccd..752c49297 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -34,12 +34,10 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: node = _find_node_for_record(record, nodes) if node: - extra = _extract_extra_metrics(node) - merged_metrics = {**record.metrics, **extra} - # Merge extra curves into existing performance_curves - existing_curves = merged_metrics.get("performance_curves", {}) - extra_curves = extra.pop("_extra_curves", {}) + extra_metrics, extra_curves = _extract_extra_metrics(node) + merged_metrics = {**record.metrics, **extra_metrics} if extra_curves: + existing_curves = merged_metrics.get("performance_curves", {}) existing_curves.update(extra_curves) merged_metrics["performance_curves"] = existing_curves enriched_records.append( @@ -67,7 +65,18 @@ def _find_node_for_record( return None -def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: +def _bench_sort_key(bench: dict) -> float: + """Sort key matching GuideLLMParser: requests_per_second.successful.mean.""" + return float( + bench.get("metrics", {}) + .get("requests_per_second", {}) + .get("successful", {}) + .get("mean", 0) + ) + + +def _extract_extra_metrics(node: TestBaseNode) -> tuple[dict[str, Any], dict[str, list]]: + """Return (scalar_metrics, extra_curves) extracted from raw benchmarks.json files.""" extra: dict[str, Any] = {} benchmarks_files = sorted( [p for p in node.artifact_paths if p.name == "benchmarks.json" @@ -75,9 +84,8 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: key=lambda p: p.name, ) if not benchmarks_files: - return extra + return extra, {} - # Collect all benchmarks across all files all_benchmarks: list[dict] = [] report_metadata: dict[str, Any] = {} report_args: dict[str, Any] = {} @@ -94,9 +102,12 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: report_args = data.get("args", {}) if not all_benchmarks: - return extra + return extra, {} - # Extract report-level metadata + # Sort by request_rate to match GuideLLMParser curve ordering + all_benchmarks.sort(key=_bench_sort_key) + + # Report-level metadata extra["guidellm_version"] = report_metadata.get("guidellm_version", "") data_str = "" @@ -108,7 +119,6 @@ def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: extra["prompt_toks"] = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" extra["output_toks"] = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" - # Compute global start/end timestamps start_times = [] end_times = [] for bench in all_benchmarks: @@ -145,8 +155,7 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: if concurrency > 0: extra["request_concurrency"] = concurrency - # Build per-benchmark extra curves for ALL dashboard metrics - # These will be merged into the existing performance_curves dict + # Per-benchmark extra curves — indices align with GuideLLM's sorted curves extra_curves: dict[str, list] = { "ttft_p1": [], "ttft_p999": [], @@ -177,22 +186,32 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: def _pct(metric_name: str, pct: str): return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + def _pct_s(metric_name: str, pct: str): + """Percentile value converted from ms to seconds.""" + v = _pct(metric_name, pct) + return v / 1000.0 if v is not None else None + def _stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("successful", {}).get(stat) + def _stat_s(metric_name: str, stat: str): + """Stat value converted from ms to seconds.""" + v = _stat(metric_name, stat) + return v / 1000.0 if v is not None else None + def _total_stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("total", {}).get(stat) request_totals = metrics.get("request_totals", {}) - extra_curves["ttft_p1"].append(_pct("time_to_first_token_ms", "p01")) - extra_curves["ttft_p999"].append(_pct("time_to_first_token_ms", "p999")) - extra_curves["ttft_mean"].append(_stat("time_to_first_token_ms", "mean")) - extra_curves["tpot_p1"].append(_pct("time_per_output_token_ms", "p01")) - extra_curves["tpot_p999"].append(_pct("time_per_output_token_ms", "p999")) - extra_curves["itl_p1"].append(_pct("inter_token_latency_ms", "p01")) - extra_curves["itl_p999"].append(_pct("inter_token_latency_ms", "p999")) - extra_curves["itl_mean"].append(_stat("inter_token_latency_ms", "mean")) + extra_curves["ttft_p1"].append(_pct_s("time_to_first_token_ms", "p01")) + extra_curves["ttft_p999"].append(_pct_s("time_to_first_token_ms", "p999")) + extra_curves["ttft_mean"].append(_stat_s("time_to_first_token_ms", "mean")) + extra_curves["tpot_p1"].append(_pct_s("time_per_output_token_ms", "p01")) + extra_curves["tpot_p999"].append(_pct_s("time_per_output_token_ms", "p999")) + extra_curves["itl_p1"].append(_pct_s("inter_token_latency_ms", "p01")) + extra_curves["itl_p999"].append(_pct_s("inter_token_latency_ms", "p999")) + extra_curves["itl_mean"].append(_stat_s("inter_token_latency_ms", "mean")) extra_curves["request_latency_min"].append(_stat("request_latency", "min")) extra_curves["request_latency_max"].append(_stat("request_latency", "max")) extra_curves["measured_rps"].append(_stat("requests_per_second", "mean")) @@ -206,5 +225,4 @@ def _total_stat(metric_name: str, stat: str): extra_curves["errored_requests"].append(request_totals.get("errored", 0)) extra_curves["intended_concurrency"].append(strategy.get("streams")) - extra["_extra_curves"] = extra_curves - return extra + return extra, extra_curves diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 4d67db20d..658462638 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -18,6 +18,13 @@ logger = logging.getLogger(__name__) +# CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. +# GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. +_SECONDS_TO_MS_COLUMNS = frozenset({ + "ttft_median", "ttft_p95", "ttft_p99", "ttft_p1", "ttft_p999", "ttft_mean", + "tpot_median", "tpot_p95", "tpot_p99", "tpot_p1", "tpot_p999", + "itl_median", "itl_p95", "itl_p99", "itl_p1", "itl_p999", "itl_mean", +}) # KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { @@ -144,6 +151,14 @@ def export_kpis_to_csv( row: dict[str, Any] = {col: "" for col in FIELDNAMES} # Populate all metric columns from KPI values row.update(metrics) + # Convert latency columns from seconds back to ms for dashboard compat + for col in _SECONDS_TO_MS_COLUMNS: + val = row.get(col) + if val not in ("", None): + try: + row[col] = float(val) * 1000.0 + except (TypeError, ValueError): + pass # Populate metadata columns from labels row["run"] = run_name row["accelerator"] = acc From 05cf21b98c654c673dfbbf9548ec940b6b855215 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 14:54:16 +0200 Subject: [PATCH 0192/1072] [llm_d] orchestration: config: platform: remove trailing " --- projects/llm_d/orchestration/config.d/platform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index a8bf91b73..4e666be30 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -8,7 +8,7 @@ cluster: namespace: labels: app.kubernetes.io/managed-by: forge - forge.openshift.io/project: llm_d" + forge.openshift.io/project: llm_d operators: openshift-cert-manager-operator: From f3f956c74d2c58cd04d810aeb7fc668370692ae2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 15:04:10 +0200 Subject: [PATCH 0193/1072] [llm_d] orchestration: render_inference_service: vllm_default_args are already applied in runtime_config.get_deployment_profile() --- projects/llm_d/orchestration/render_inference_service.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 79e433ee0..29c254333 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -173,12 +173,9 @@ def _build_vllm_additional_args( String suitable for VLLM_ADDITIONAL_ARGS environment variable """ - vllm_default_args = _build_vllm_args( - config.project.get_config("deployments.defaults.vllm_extra.args") - ) - vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra.args", {})) + vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra", {}).get("args")) - return " ".join(vllm_default_args + vllm_deploy_args) + return " ".join(vllm_deploy_args) def _render_standard_deployment( From 3c629b86edeb0c5e74dc5d27e6bce8cb4613c012 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 24 Jul 2026 10:35:02 +0200 Subject: [PATCH 0194/1072] [projects] Fix ruff in MD files --- AGENTS.md | 2 + docs/orchestration/vaults.md | 24 ++++---- docs/toolbox/dsl.md | 6 +- docs/toolbox/toolbox-command-guide.md | 58 +++++++++++-------- .../cluster/toolbox/rebuild_image/README.md | 6 +- projects/core/CHANGELOG-dsl.md | 1 + projects/core/notifications/README.md | 2 +- projects/llm_d/docs/PD_TESTING.md | 8 +-- projects/skeleton/README.md | 42 ++++---------- 9 files changed, 70 insertions(+), 79 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4ba19d7a0..ddeaaaa52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,8 +55,10 @@ def apply_secret(manifest: dict) -> None: """Apply a Secret manifest via stdin — nothing written to disk.""" import yaml from projects.core.dsl.utils.k8s import oc + oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False)) + # GOOD: Guard before using oc_apply def safe_oc_apply(artifact_path, manifest): if manifest.get("kind") == "Secret": diff --git a/docs/orchestration/vaults.md b/docs/orchestration/vaults.md index edef7ad06..44a8928ce 100644 --- a/docs/orchestration/vaults.md +++ b/docs/orchestration/vaults.md @@ -116,7 +116,7 @@ from projects.core.library import vault vault.disable_strict_validation() # All subsequent vault operations will use warning mode -vault.init(['psap-forge-ci']) # Uses non-strict validation +vault.init(["psap-forge-ci"]) # Uses non-strict validation ``` **When to use non-strict mode:** @@ -140,10 +140,10 @@ vault.disable_strict_validation() # Optional: disable strict mode vault_manager = get_vault_manager() # Validate a specific vault -is_valid = vault_manager.validate_vault('psap-forge-ci') +is_valid = vault_manager.validate_vault("psap-forge-ci") # Validate all project vaults -is_valid = vault_manager.validate_project_vaults('llm_d') +is_valid = vault_manager.validate_project_vaults("llm_d") # Validate all defined vaults is_valid = vault_manager.validate_all_vaults() @@ -155,12 +155,12 @@ is_valid = vault_manager.validate_all_vaults() from projects.core.library.vault import get_vault_content_path # Get full path to vault content -kubeconfig_path = get_vault_content_path('psap-forge-ci', 'kubeconfig') +kubeconfig_path = get_vault_content_path("psap-forge-ci", "kubeconfig") # Returns: Path('/secrets/ci/kubeconfig') # Access via vault object -vault = get_vault_manager().get_vault('psap-forge-ci') -kubeconfig_path = vault.content['kubeconfig'].file_path +vault = get_vault_manager().get_vault("psap-forge-ci") +kubeconfig_path = vault.content["kubeconfig"].file_path # Returns: Path('/secrets/ci/kubeconfig') ``` @@ -239,7 +239,7 @@ In `projects/my-project/orchestration/vaults.yaml`: from projects.core.library.vault import validate_project_vaults # Validate the new vault (uses global strict validation setting) -validate_project_vaults('my-project') +validate_project_vaults("my-project") ``` ### 6. Expose the env var to the `forge_launcher` @@ -265,7 +265,7 @@ from projects.core.library import vault vault.init() # Initialize with specific vaults only -vault.init(['psap-forge-ci', 'forge-llm-d']) +vault.init(["psap-forge-ci", "forge-llm-d"]) ``` ### Runtime Usage @@ -277,13 +277,13 @@ from projects.core.library.vault import get_vault_manager vault_manager = get_vault_manager() # Validate project vaults before execution -if not vault_manager.validate_project_vaults('llm_d'): +if not vault_manager.validate_project_vaults("llm_d"): raise RuntimeError("Vault validation failed") # Get paths to vault content -vault = vault_manager.get_vault('forge-llm-d') -hf_token_path = vault.content['hf_token'].file_path -pull_secret_path = vault.content['rhoai_pull_secret'].file_path +vault = vault_manager.get_vault("forge-llm-d") +hf_token_path = vault.content["hf_token"].file_path +pull_secret_path = vault.content["rhoai_pull_secret"].file_path ``` ## Error Handling diff --git a/docs/toolbox/dsl.md b/docs/toolbox/dsl.md index 101326c51..54246707c 100644 --- a/docs/toolbox/dsl.md +++ b/docs/toolbox/dsl.md @@ -37,8 +37,7 @@ def ensure_project(args, ctx): ```python @when(lambda: some_other_task.status.return_value is True) @task -def follow_up(args, ctx): - ... +def follow_up(args, ctx): ... ``` Because the condition is called with **no arguments**, anything dynamic must come from a **closure**, module-level state, or another task’s `.status.return_value` (see `TaskResult` in `script_manager.py`). @@ -69,8 +68,7 @@ def wait_until_ready(args, ctx): ```python @retry(attempts=5, delay=2, backoff=1.5, retry_on_exceptions=True) @task -def call_flaky_api(args, ctx): - ... +def call_flaky_api(args, ctx): ... ``` If all attempts fail, `RetryFailure` is raised (wrapped in `TaskExecutionError` during `execute_tasks`, with the underlying `RetryFailure` available as `TaskExecutionError.__cause__` when that applies). diff --git a/docs/toolbox/toolbox-command-guide.md b/docs/toolbox/toolbox-command-guide.md index 3a6f688ee..37c15b24c 100644 --- a/docs/toolbox/toolbox-command-guide.md +++ b/docs/toolbox/toolbox-command-guide.md @@ -174,6 +174,7 @@ def capture_resources(args, ctx): shell.run("oc delete pod mypod", check=False) return "Cleanup completed" + @always @task def cleanup_resources(args, ctx): @@ -195,10 +196,13 @@ shell.run("oc get pods") result = shell.run("oc get pods -o json") if result.success: import json + pods = json.loads(result.stdout) # Save output to file -shell.run("oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt") +shell.run( + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt" +) # Don't fail on error shell.run("oc delete pod optional-pod", check=False) @@ -278,21 +282,16 @@ def capture_pod_status(args, ctx): # Human readable shell.run( - "oc describe pod mypod", - stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" ) # Machine readable shell.run( - "oc get pod mypod -o yaml", - stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" + "oc get pod mypod -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" ) # For large scale tests, also capture JSON - shell.run( - "oc get pods -o json", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json" - ) + shell.run("oc get pods -o json", stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json") ``` #### Source Directory (`src/`) @@ -356,7 +355,7 @@ def run(password: str): # DON'T DO THIS! #### ❌ BAD: Secrets in environment variables ```python -os.environ['SECRET_TOKEN'] = token # DON'T DO THIS! +os.environ["SECRET_TOKEN"] = token # DON'T DO THIS! # Environment variables appear in logs and process lists ``` @@ -490,10 +489,12 @@ Use descriptive task names that explain the **purpose**, not just the action: def wait_for_operator_to_be_ready(args, ctx): """Wait for operator deployment to reach ready state""" + @task def capture_failed_pod_logs(args, ctx): """Collect logs from pods that failed to start""" + @task def verify_service_endpoints_are_available(args, ctx): """Check that service endpoints are responding to requests""" @@ -523,7 +524,7 @@ def wait_for_resource_ready(args, ctx): shell.run( f"oc describe myresource {args.resource_name}", stdout_dest=args.artifact_dir / "artifacts" / "failed-resource-describe.txt", - check=False + check=False, ) raise RuntimeError(f"Failed to query resource {args.resource_name}: {result.stderr}") @@ -541,13 +542,17 @@ def capture_cluster_state(args, ctx): """Capture cluster state for post-mortem analysis""" # Capture even if tests failed - shell.run("oc get pods --all-namespaces", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", - check=False) + shell.run( + "oc get pods --all-namespaces", + stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", + check=False, + ) - shell.run("oc get events --sort-by='.lastTimestamp'", - stdout_dest=args.artifact_dir / "artifacts" / "events.txt", - check=False) + shell.run( + "oc get events --sort-by='.lastTimestamp'", + stdout_dest=args.artifact_dir / "artifacts" / "events.txt", + check=False, + ) return "Cluster state captured" ``` @@ -560,9 +565,8 @@ def capture_cluster_state(args, ctx): ```python #!/usr/bin/env python3 -from projects.core.dsl import ( - always, entrypoint, execute_tasks, retry, shell, task, template -) +from projects.core.dsl import always, entrypoint, execute_tasks, retry, shell, task, template + @entrypoint def run( @@ -585,6 +589,7 @@ def run( """ return execute_tasks(locals()) + @task def validate_inputs(args, ctx): """Validate input parameters""" @@ -598,6 +603,7 @@ def validate_inputs(args, ctx): return "Inputs validated" + @task def setup_directories(args, ctx): """Create artifact directories""" @@ -606,6 +612,7 @@ def setup_directories(args, ctx): shell.mkdir("src") return "Directories created" + @task def verify_namespace_exists(args, ctx): """Ensure target namespace exists""" @@ -616,6 +623,7 @@ def verify_namespace_exists(args, ctx): return f"Namespace {args.namespace} verified" + @task def create_deployment_manifest(args, ctx): """Generate deployment manifest""" @@ -625,6 +633,7 @@ def create_deployment_manifest(args, ctx): return f"Deployment manifest created: {manifest_file}" + @task def apply_deployment(args, ctx): """Apply the deployment to cluster""" @@ -634,6 +643,7 @@ def apply_deployment(args, ctx): return f"Deployment {args.service_name} applied" + @retry(attempts=10, delay=5) @task def wait_for_deployment_ready(args, ctx): @@ -642,7 +652,7 @@ def wait_for_deployment_ready(args, ctx): result = shell.run( f"oc get deployment {args.service_name} -n {args.namespace} " f"-o jsonpath='{{.status.readyReplicas}}'", - check=False + check=False, ) if not result.success: @@ -654,6 +664,7 @@ def wait_for_deployment_ready(args, ctx): return False # Retry + @always @task def capture_deployment_status(args, ctx): @@ -663,18 +674,19 @@ def capture_deployment_status(args, ctx): shell.run( f"oc describe deployment {args.service_name} -n {args.namespace}", stdout_dest=args.artifact_dir / "artifacts" / "deployment-describe.txt", - check=False + check=False, ) # Capture pod status shell.run( f"oc get pods -l app={args.service_name} -n {args.namespace} -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pods-status.yaml", - check=False + check=False, ) return "Deployment status captured" + if __name__ == "__main__": run.main() ``` diff --git a/projects/cluster/toolbox/rebuild_image/README.md b/projects/cluster/toolbox/rebuild_image/README.md index 324f2ea26..062defce5 100644 --- a/projects/cluster/toolbox/rebuild_image/README.md +++ b/projects/cluster/toolbox/rebuild_image/README.md @@ -43,11 +43,7 @@ Can be imported and used programmatically: ```python from projects.cluster.toolbox.rebuild_image.main import run as rebuild_image -result = rebuild_image( - build_name="my-build", - namespace="my-namespace", - timeout_minutes=30 -) +result = rebuild_image(build_name="my-build", namespace="my-namespace", timeout_minutes=30) ``` ## Common Use Cases diff --git a/projects/core/CHANGELOG-dsl.md b/projects/core/CHANGELOG-dsl.md index b3f69351c..a3c7cd951 100644 --- a/projects/core/CHANGELOG-dsl.md +++ b/projects/core/CHANGELOG-dsl.md @@ -42,6 +42,7 @@ ```python from projects.core.dsl import EarlyReturn + @task def check_operator_deployed(args, ctx): if operator_exists(): diff --git a/projects/core/notifications/README.md b/projects/core/notifications/README.md index cc7442565..e3bf9831b 100644 --- a/projects/core/notifications/README.md +++ b/projects/core/notifications/README.md @@ -59,7 +59,7 @@ success = send_notification( message="🟢 Test completed successfully\nResults: https://example.com/results", github=True, slack=False, - dry_run=False + dry_run=False, ) ``` diff --git a/projects/llm_d/docs/PD_TESTING.md b/projects/llm_d/docs/PD_TESTING.md index b62c752d5..aeb2c3d8d 100644 --- a/projects/llm_d/docs/PD_TESTING.md +++ b/projects/llm_d/docs/PD_TESTING.md @@ -152,7 +152,7 @@ pd_config = runtime_config.get_pd_config() # Get individual P/D parameters prefill_pods = runtime_config.get_prefill_pod_count() # Returns: 2 -decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 +decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 scheduler_config = runtime_config.get_scheduler_config() # Returns: 'high-throughput' # Check if current deployment is P/D @@ -272,13 +272,13 @@ if not match: # For prefill_tensor_parallelism match = re.search(r"p\.tp(\d+)", profile_name) # p.tp1 → 1 -# For decode_pods (main container replicas) -match = re.search(r"d\.x(\d+)", profile_name) # d.x2 → 2 +# For decode_pods (main container replicas) +match = re.search(r"d\.x(\d+)", profile_name) # d.x2 → 2 if not match: match = re.search(r"(? Date: Fri, 24 Jul 2026 10:47:37 +0200 Subject: [PATCH 0195/1072] ruff: pin the version to 0.16.0 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcab8c11a..cc1dcab29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.16.0 hooks: - id: ruff args: [--fix] diff --git a/pyproject.toml b/pyproject.toml index 400f0eb8f..4faa72b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dev = [ "pytest>=7.4.0", "pytest-asyncio>=0.21.1", "black>=23.7.0", - "ruff>=0.0.287", + "ruff==0.16.0", "pre-commit>=4.0", ] docs = [ From ece24b94b0eb94d580b3fb583979e78090fce49e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 26 Jul 2026 15:22:10 +0300 Subject: [PATCH 0196/1072] cluster: add capture_prometheus toolbox for extracting time-bounded metrics Adds a new toolbox under projects/cluster/toolbox/capture_prometheus that extracts Prometheus metrics for a specific time window from the cluster's monitoring stack and saves them as a compressed OpenMetrics file. Uses a symlink trick to scan only WAL + chunks_head (~1-2 GB) instead of the full TSDB, making extraction fast (~2 min for 20 min of metrics). Currently limited to tests under 2 hours. Co-authored-by: Cursor --- .../toolbox/capture_prometheus/README.md | 80 +++++++ .../toolbox/capture_prometheus/main.py | 211 ++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 projects/cluster/toolbox/capture_prometheus/README.md create mode 100644 projects/cluster/toolbox/capture_prometheus/main.py diff --git a/projects/cluster/toolbox/capture_prometheus/README.md b/projects/cluster/toolbox/capture_prometheus/README.md new file mode 100644 index 000000000..c59eec15d --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/README.md @@ -0,0 +1,80 @@ +# capture_prometheus + +Extracts cluster Prometheus metrics for a specific time window and saves them as a compressed OpenMetrics formated file. The output can later be imported into a local Prometheus instance for offline querying of any metric. + +## How it works + +1. **Symlink trick** — Creates a temporary directory on the Prometheus PVC containing only symlinks to `wal/` and `chunks_head/`. This limits `promtool` to scanning the recent in-memory data (~1-2 GB) instead of the full TSDB (which can be 20+ GB of persistent blocks). +2. **Time-filtered dump** — Runs `promtool tsdb dump-openmetrics --min-time=X --max-time=Y` inside the Prometheus container against the temp directory. Only samples within the requested window are emitted. +3. **Compress and copy** — The output is gzipped on the pod, then copied to the specified output directory via `oc cp`. +4. **Cleanup** — Removes the temp directory and compressed file from the pod. + + + +## Limitations + +- **Max test duration: 2 hours.** The WAL + chunks_head holds approximately the last 2 hours of un-compacted data. Tests longer than 2 hours will fail with an error. Support for including persistent blocks that overlap with the test window can be added later. +- **Data granularity:** The dump includes all metrics scraped by cluster monitoring (typically every 30s). For a 20-minute test this produces ~24M data points / ~130MB compressed. + + + +## Usage + + + +### Standalone (CLI) + +```bash +cd forge-prom/forge +python3 -m projects.cluster.toolbox.capture_prometheus.main \ + "2026-07-26T10:00:00+00:00" \ + "2026-07-26T10:20:00+00:00" \ + /path/to/output +``` + + + +### From orchestration + +```python +from projects.cluster.toolbox.capture_prometheus import main as capture_prom + +prometheus_dir = env.ARTIFACT_DIR / "prometheus" +prometheus_dir.mkdir(parents=True, exist_ok=True) + +capture_prom.run( + start_time=test_start_time, + end_time=test_end_time, + output_dir=str(prometheus_dir), +) +``` + + + +## Querying the data locally + +To analyze the captured metrics offline: + +1. **Decompress** the archive to get a plain OpenMetrics text file. +2. **Import** the file into a Prometheus TSDB using `promtool tsdb create-blocks-from openmetrics`. +3. **Run a Prometheus container** (Docker, Podman, or any OCI runtime) with: + - The imported TSDB directory mounted at `/prometheus` + - A minimal config file (an empty `global: {}` is sufficient since no scraping is needed) + - Retention set high enough that the data won't be garbage-collected (e.g. `--storage.tsdb.retention.time=365d`) +4. **Open the Prometheus UI** and query any metric using the time range from `metadata.json`. + +The output file (`metrics.openmetrics.gz`) follows the standard [OpenMetrics](https://openmetrics.io/) format and is compatible with any tool that can ingest it (Prometheus, Grafana via Prometheus data source, etc.). + +## Parameters + + +| Parameter | Required | Default | Description | +| ------------- | -------- | ---------------------- | ------------------------------------------------ | +| `start_time` | yes | — | Start of capture window (ISO 8601, UTC) | +| `end_time` | yes | — | End of capture window (ISO 8601, UTC) | +| `output_dir` | yes | — | Directory to write `metrics.openmetrics.gz` into | +| `--pod-name` | no | `prometheus-k8s-0` | Prometheus pod name | +| `--namespace` | no | `openshift-monitoring` | Namespace of Prometheus | +| `--container` | no | `prometheus` | Container name in the pod | + + diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py new file mode 100644 index 000000000..e290883f9 --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 + +""" +Capture Prometheus TSDB Toolbox + +Extracts cluster Prometheus metrics for a specific time window and saves them +as a compressed OpenMetrics archive. The output can later be imported into a +local Prometheus instance for offline querying. + +Can be run standalone: + python -m projects.cluster.toolbox.capture_prometheus.main \\ + --start-time "2026-07-26T10:00:00+00:00" \\ + --end-time "2026-07-26T10:20:00+00:00" \\ + --output-dir /path/to/output +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from pathlib import Path + +from projects.core.dsl import always, entrypoint, execute_tasks, shell, task + +logger = logging.getLogger("DSL") + +PROMETHEUS_NAMESPACE = "openshift-monitoring" +PROMETHEUS_POD = "prometheus-k8s-0" +PROMETHEUS_CONTAINER = "prometheus" +TSDB_PATH = "/prometheus" + + +@entrypoint +def run( + start_time: datetime, + end_time: datetime, + output_dir: str | Path, + *, + pod_name: str = PROMETHEUS_POD, + namespace: str = PROMETHEUS_NAMESPACE, + container: str = PROMETHEUS_CONTAINER, +) -> int: + """ + Capture Prometheus metrics for a specific time window. + + Extracts all metrics between start_time and end_time from the cluster's + Prometheus TSDB and writes them as a compressed OpenMetrics file to output_dir. + + Args: + start_time: Start of the capture window (UTC) + end_time: End of the capture window (UTC) + output_dir: Directory to write the metrics archive into + pod_name: Prometheus pod to extract from + namespace: Namespace where Prometheus runs + container: Container name within the Prometheus pod + """ + execute_tasks(locals()) + return 0 + + +def _parse_time(value) -> datetime: + """Parse a datetime from string (ISO format) or pass through if already datetime.""" + if isinstance(value, datetime): + return value + dt = datetime.fromisoformat(str(value)) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + + +@task +def validate_parameters(args, ctx): + """Validate inputs and compute time bounds.""" + + ctx.start_time = _parse_time(args.start_time) + ctx.end_time = _parse_time(args.end_time) + + if ctx.end_time <= ctx.start_time: + raise ValueError("end_time must be after start_time") + + max_duration_seconds = 2 * 3600 + duration = (ctx.end_time - ctx.start_time).total_seconds() + if duration > max_duration_seconds: + raise ValueError( + f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " + f"Tests longer than 2 hours require persistent block handling (not yet implemented)." + ) + + ctx.output_dir = Path(args.output_dir) + ctx.output_dir.mkdir(parents=True, exist_ok=True) + ctx.start_ms = int(ctx.start_time.timestamp() * 1000) + ctx.end_ms = int(ctx.end_time.timestamp() * 1000) + ctx.duration_seconds = int((ctx.end_time - ctx.start_time).total_seconds()) + + ctx.namespace = args.namespace or PROMETHEUS_NAMESPACE + ctx.pod_name = args.pod_name or PROMETHEUS_POD + ctx.container = args.container or PROMETHEUS_CONTAINER + + return ( + f"Capture window: {ctx.start_time.isoformat()} → {ctx.end_time.isoformat()} " + f"({ctx.duration_seconds}s)" + ) + + +@task +def validate_prometheus_pod(args, ctx): + """Verify the Prometheus pod is running and accessible.""" + + result = shell.run( + f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", + ) + + phase = result.stdout.strip() + if phase != "Running": + raise RuntimeError( + f"Prometheus pod {ctx.pod_name} is in phase '{phase}', expected 'Running'" + ) + + return f"Prometheus pod {ctx.pod_name} is running" + + +@task +def create_temp_tsdb_dir(args, ctx): + """Create a temp directory on the PVC with symlinks to WAL and chunks_head only. + + This limits promtool to scanning only recent in-memory data (~1-2 GB) + instead of the full TSDB which can be tens of GB. + """ + + ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" + + shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"sh -c '" + f"mkdir -p {ctx.temp_dir} && " + f"ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal && " + f"ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" + f"'", + ) + + return f"Created temp TSDB dir at {ctx.temp_dir}" + + +@task +def dump_metrics(args, ctx): + """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" + + ctx.remote_output = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}.gz" + + shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"sh -c '" + f"promtool tsdb dump-openmetrics " + f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " + f"{ctx.temp_dir} | gzip > {ctx.remote_output}" + f"'", + ) + + size_result = shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + f"stat -c %s {ctx.remote_output}", + check=False, + ) + + ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 + + return f"Dumped metrics ({ctx.archive_size_bytes} bytes compressed)" + + +@task +def copy_to_output(args, ctx): + """Copy the compressed metrics file from the pod to the output directory.""" + + ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" + + shell.run( + f"oc cp {ctx.namespace}/{ctx.pod_name}:{ctx.remote_output} " + f"{ctx.output_file} -c {ctx.container}", + ) + + return f"Copied metrics to {ctx.output_file}" + + +@always +@task +def cleanup_pod(args, ctx): + """Remove temp files from the Prometheus pod.""" + + namespace = getattr(ctx, "namespace", None) or PROMETHEUS_NAMESPACE + pod_name = getattr(ctx, "pod_name", None) or PROMETHEUS_POD + container = getattr(ctx, "container", None) or PROMETHEUS_CONTAINER + temp_dir = getattr(ctx, "temp_dir", None) + remote_output = getattr(ctx, "remote_output", None) + + if temp_dir: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -rf {temp_dir}", + check=False, + ) + + if remote_output: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_output}", + check=False, + ) + + return "Cleaned up temp files on Prometheus pod" + + +if __name__ == "__main__": + run.main() From 6502de3883c8981e4086d37d230001c959ebbfd2 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 26 Jul 2026 18:49:57 +0300 Subject: [PATCH 0197/1072] addres few code rabbit issues --- .../toolbox/capture_prometheus/main.py | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index e290883f9..da18d5368 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -59,13 +59,18 @@ def run( def _parse_time(value) -> datetime: - """Parse a datetime from string (ISO format) or pass through if already datetime.""" + """Parse a datetime from string (ISO format) or pass through if already datetime. + + Naive datetimes are assumed UTC. Aware datetimes are normalized to UTC. + """ if isinstance(value, datetime): - return value + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) dt = datetime.fromisoformat(str(value)) if dt.tzinfo is None: dt = dt.replace(tzinfo=UTC) - return dt + return dt.astimezone(UTC) @task @@ -81,10 +86,13 @@ def validate_parameters(args, ctx): max_duration_seconds = 2 * 3600 duration = (ctx.end_time - ctx.start_time).total_seconds() if duration > max_duration_seconds: - raise ValueError( - f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " - f"Tests longer than 2 hours require persistent block handling (not yet implemented)." + logger.warning( + "Test duration (%ds) exceeds the 2-hour WAL window. " + "Skipping Prometheus capture (persistent block handling not yet implemented).", + int(duration), ) + ctx.skip_capture = True + return f"SKIPPED: duration {int(duration)}s exceeds 2-hour WAL window" ctx.output_dir = Path(args.output_dir) ctx.output_dir.mkdir(parents=True, exist_ok=True) @@ -106,6 +114,9 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + result = shell.run( f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", ) @@ -127,6 +138,9 @@ def create_temp_tsdb_dir(args, ctx): instead of the full TSDB which can be tens of GB. """ + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" shell.run( @@ -145,14 +159,18 @@ def create_temp_tsdb_dir(args, ctx): def dump_metrics(args, ctx): """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" - ctx.remote_output = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}.gz" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + + ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" + ctx.remote_output = f"{ctx.remote_raw}.gz" shell.run( f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " f"sh -c '" f"promtool tsdb dump-openmetrics " f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " - f"{ctx.temp_dir} | gzip > {ctx.remote_output}" + f"{ctx.temp_dir} > {ctx.remote_raw} && gzip -f {ctx.remote_raw}" f"'", ) @@ -171,6 +189,9 @@ def dump_metrics(args, ctx): def copy_to_output(args, ctx): """Copy the compressed metrics file from the pod to the output directory.""" + if getattr(ctx, "skip_capture", False): + return "SKIPPED" + ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" shell.run( @@ -204,6 +225,13 @@ def cleanup_pod(args, ctx): check=False, ) + remote_raw = getattr(ctx, "remote_raw", None) + if remote_raw: + shell.run( + f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_raw}", + check=False, + ) + return "Cleaned up temp files on Prometheus pod" From 343b78c7dfc83b8041b90d8b057aa8a61215eea8 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 27 Jul 2026 09:31:22 +0300 Subject: [PATCH 0198/1072] address kevin's comments, as well as a small fix to the core cli entry point to correctly parse defualt values --- .../toolbox/capture_prometheus/README.md | 3 +- .../toolbox/capture_prometheus/main.py | 107 +++++++----------- projects/core/dsl/cli.py | 1 + 3 files changed, 44 insertions(+), 67 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/README.md b/projects/cluster/toolbox/capture_prometheus/README.md index c59eec15d..63c8d09ce 100644 --- a/projects/cluster/toolbox/capture_prometheus/README.md +++ b/projects/cluster/toolbox/capture_prometheus/README.md @@ -25,8 +25,7 @@ Extracts cluster Prometheus metrics for a specific time window and saves them as ### Standalone (CLI) ```bash -cd forge-prom/forge -python3 -m projects.cluster.toolbox.capture_prometheus.main \ +./bin/run_toolbox cluster capture_prometheus \ "2026-07-26T10:00:00+00:00" \ "2026-07-26T10:20:00+00:00" \ /path/to/output diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index da18d5368..f2c5469f7 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -8,10 +8,10 @@ local Prometheus instance for offline querying. Can be run standalone: - python -m projects.cluster.toolbox.capture_prometheus.main \\ - --start-time "2026-07-26T10:00:00+00:00" \\ - --end-time "2026-07-26T10:20:00+00:00" \\ - --output-dir /path/to/output + ./bin/run_toolbox cluster capture_prometheus \\ + "2026-07-26T10:00:00+00:00" \\ + "2026-07-26T10:20:00+00:00" \\ + /path/to/output """ from __future__ import annotations @@ -58,6 +58,21 @@ def run( return 0 +def _oc_exec(ctx, command: str, **kwargs): + """Run a command inside the Prometheus container via oc exec.""" + return shell.run( + f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- {command}", + **kwargs, + ) + + +def _oc_cp_from_pod(ctx, remote_path: str, local_path): + """Copy a file from the Prometheus container to a local path.""" + return shell.run( + f"oc cp {ctx.namespace}/{ctx.pod_name}:{remote_path} {local_path} -c {ctx.container}", + ) + + def _parse_time(value) -> datetime: """Parse a datetime from string (ISO format) or pass through if already datetime. @@ -86,13 +101,10 @@ def validate_parameters(args, ctx): max_duration_seconds = 2 * 3600 duration = (ctx.end_time - ctx.start_time).total_seconds() if duration > max_duration_seconds: - logger.warning( - "Test duration (%ds) exceeds the 2-hour WAL window. " - "Skipping Prometheus capture (persistent block handling not yet implemented).", - int(duration), + raise ValueError( + f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " + f"Tests longer than 2 hours require persistent block handling (not yet implemented)." ) - ctx.skip_capture = True - return f"SKIPPED: duration {int(duration)}s exceeds 2-hour WAL window" ctx.output_dir = Path(args.output_dir) ctx.output_dir.mkdir(parents=True, exist_ok=True) @@ -100,9 +112,9 @@ def validate_parameters(args, ctx): ctx.end_ms = int(ctx.end_time.timestamp() * 1000) ctx.duration_seconds = int((ctx.end_time - ctx.start_time).total_seconds()) - ctx.namespace = args.namespace or PROMETHEUS_NAMESPACE - ctx.pod_name = args.pod_name or PROMETHEUS_POD - ctx.container = args.container or PROMETHEUS_CONTAINER + ctx.namespace = args.namespace + ctx.pod_name = args.pod_name + ctx.container = args.container return ( f"Capture window: {ctx.start_time.isoformat()} → {ctx.end_time.isoformat()} " @@ -114,9 +126,6 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - result = shell.run( f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", ) @@ -138,17 +147,14 @@ def create_temp_tsdb_dir(args, ctx): instead of the full TSDB which can be tens of GB. """ - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" - shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + _oc_exec( + ctx, f"sh -c '" - f"mkdir -p {ctx.temp_dir} && " - f"ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal && " - f"ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" + f"mkdir -p {ctx.temp_dir}" + f" && ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal" + f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" f"'", ) @@ -159,26 +165,20 @@ def create_temp_tsdb_dir(args, ctx): def dump_metrics(args, ctx): """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" ctx.remote_output = f"{ctx.remote_raw}.gz" - shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " + _oc_exec( + ctx, f"sh -c '" - f"promtool tsdb dump-openmetrics " - f"--min-time={ctx.start_ms} --max-time={ctx.end_ms} " - f"{ctx.temp_dir} > {ctx.remote_raw} && gzip -f {ctx.remote_raw}" + f"promtool tsdb dump-openmetrics" + f" --min-time={ctx.start_ms} --max-time={ctx.end_ms}" + f" {ctx.temp_dir} > {ctx.remote_raw}" + f" && gzip -f {ctx.remote_raw}" f"'", ) - size_result = shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- " - f"stat -c %s {ctx.remote_output}", - check=False, - ) + size_result = _oc_exec(ctx, f"stat -c %s {ctx.remote_output}", check=False) ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 @@ -189,15 +189,9 @@ def dump_metrics(args, ctx): def copy_to_output(args, ctx): """Copy the compressed metrics file from the pod to the output directory.""" - if getattr(ctx, "skip_capture", False): - return "SKIPPED" - ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" - shell.run( - f"oc cp {ctx.namespace}/{ctx.pod_name}:{ctx.remote_output} " - f"{ctx.output_file} -c {ctx.container}", - ) + _oc_cp_from_pod(ctx, ctx.remote_output, ctx.output_file) return f"Copied metrics to {ctx.output_file}" @@ -207,30 +201,13 @@ def copy_to_output(args, ctx): def cleanup_pod(args, ctx): """Remove temp files from the Prometheus pod.""" - namespace = getattr(ctx, "namespace", None) or PROMETHEUS_NAMESPACE - pod_name = getattr(ctx, "pod_name", None) or PROMETHEUS_POD - container = getattr(ctx, "container", None) or PROMETHEUS_CONTAINER temp_dir = getattr(ctx, "temp_dir", None) remote_output = getattr(ctx, "remote_output", None) - - if temp_dir: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -rf {temp_dir}", - check=False, - ) - - if remote_output: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_output}", - check=False, - ) - remote_raw = getattr(ctx, "remote_raw", None) - if remote_raw: - shell.run( - f"oc -n {namespace} exec {pod_name} -c {container} -- rm -f {remote_raw}", - check=False, - ) + + for path in [temp_dir, remote_output, remote_raw]: + if path: + _oc_exec(ctx, f"rm -rf {path}", check=False) return "Cleaned up temp files on Prometheus pod" diff --git a/projects/core/dsl/cli.py b/projects/core/dsl/cli.py index cc1617a47..fc902637b 100644 --- a/projects/core/dsl/cli.py +++ b/projects/core/dsl/cli.py @@ -216,6 +216,7 @@ def resolve_annotation(annotation): cli_name, type=arg_type, required=not has_default, + default=param.default if has_default else None, dest=param_name, help=help_text, ) From d5355c5c968e61aa791279f26a2332724f3067f0 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 27 Jul 2026 10:21:02 +0300 Subject: [PATCH 0199/1072] refactor: move oc_exec/oc_cp_from_pod to shared k8s utils library Address reviewer feedback to make pod exec and copy helpers reusable across toolboxes instead of keeping them private to capture_prometheus. Co-authored-by: Cursor --- .../toolbox/capture_prometheus/main.py | 56 +++++++++---------- projects/core/dsl/utils/k8s.py | 53 ++++++++++++++++++ 2 files changed, 80 insertions(+), 29 deletions(-) diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py index f2c5469f7..8fcd7949d 100644 --- a/projects/cluster/toolbox/capture_prometheus/main.py +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -20,7 +20,8 @@ from datetime import UTC, datetime from pathlib import Path -from projects.core.dsl import always, entrypoint, execute_tasks, shell, task +from projects.core.dsl import always, entrypoint, execute_tasks, task +from projects.core.dsl.utils.k8s import oc, oc_cp_from_pod, oc_exec logger = logging.getLogger("DSL") @@ -58,19 +59,9 @@ def run( return 0 -def _oc_exec(ctx, command: str, **kwargs): - """Run a command inside the Prometheus container via oc exec.""" - return shell.run( - f"oc -n {ctx.namespace} exec {ctx.pod_name} -c {ctx.container} -- {command}", - **kwargs, - ) - - -def _oc_cp_from_pod(ctx, remote_path: str, local_path): - """Copy a file from the Prometheus container to a local path.""" - return shell.run( - f"oc cp {ctx.namespace}/{ctx.pod_name}:{remote_path} {local_path} -c {ctx.container}", - ) +def _pod_kwargs(ctx) -> dict: + """Return the common namespace/pod/container kwargs for oc_exec/oc_cp_from_pod.""" + return {"namespace": ctx.namespace, "pod": ctx.pod_name, "container": ctx.container} def _parse_time(value) -> datetime: @@ -126,8 +117,14 @@ def validate_parameters(args, ctx): def validate_prometheus_pod(args, ctx): """Verify the Prometheus pod is running and accessible.""" - result = shell.run( - f"oc -n {ctx.namespace} get pod {ctx.pod_name} -o jsonpath='{{.status.phase}}'", + result = oc( + "-n", + ctx.namespace, + "get", + "pod", + ctx.pod_name, + "-o", + "jsonpath={.status.phase}", ) phase = result.stdout.strip() @@ -149,13 +146,13 @@ def create_temp_tsdb_dir(args, ctx): ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" - _oc_exec( - ctx, - f"sh -c '" + oc_exec( + "sh", + "-c", f"mkdir -p {ctx.temp_dir}" f" && ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal" - f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head" - f"'", + f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head", + **_pod_kwargs(ctx), ) return f"Created temp TSDB dir at {ctx.temp_dir}" @@ -168,17 +165,17 @@ def dump_metrics(args, ctx): ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" ctx.remote_output = f"{ctx.remote_raw}.gz" - _oc_exec( - ctx, - f"sh -c '" + oc_exec( + "sh", + "-c", f"promtool tsdb dump-openmetrics" f" --min-time={ctx.start_ms} --max-time={ctx.end_ms}" f" {ctx.temp_dir} > {ctx.remote_raw}" - f" && gzip -f {ctx.remote_raw}" - f"'", + f" && gzip -f {ctx.remote_raw}", + **_pod_kwargs(ctx), ) - size_result = _oc_exec(ctx, f"stat -c %s {ctx.remote_output}", check=False) + size_result = oc_exec("stat", "-c", "%s", ctx.remote_output, check=False, **_pod_kwargs(ctx)) ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 @@ -191,7 +188,7 @@ def copy_to_output(args, ctx): ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" - _oc_cp_from_pod(ctx, ctx.remote_output, ctx.output_file) + oc_cp_from_pod(ctx.remote_output, ctx.output_file, **_pod_kwargs(ctx)) return f"Copied metrics to {ctx.output_file}" @@ -204,10 +201,11 @@ def cleanup_pod(args, ctx): temp_dir = getattr(ctx, "temp_dir", None) remote_output = getattr(ctx, "remote_output", None) remote_raw = getattr(ctx, "remote_raw", None) + kwargs = _pod_kwargs(ctx) for path in [temp_dir, remote_output, remote_raw]: if path: - _oc_exec(ctx, f"rm -rf {path}", check=False) + oc_exec("rm", "-rf", path, check=False, **kwargs) return "Cleaned up temp files on Prometheus pod" diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 82a283384..381dd1bed 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -289,6 +289,59 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: ) +def oc_exec( + *command: str, + namespace: str, + pod: str, + container: str | None = None, + **kwargs, +) -> shell.CommandResult: + """Exec a command inside a pod container. + + Args: + *command: Command and arguments to run in the pod + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + **kwargs: Additional keyword arguments passed to oc() + + Returns: + CommandResult with execution details + """ + args = ["-n", namespace, "exec", pod] + if container: + args.extend(["-c", container]) + args.append("--") + args.extend(command) + return oc(*args, **kwargs) + + +def oc_cp_from_pod( + remote_path: str, + local_path: str | Path, + *, + namespace: str, + pod: str, + container: str | None = None, +) -> shell.CommandResult: + """Copy a file from a pod to a local path. + + Args: + remote_path: Absolute path inside the pod + local_path: Local destination path + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + + Returns: + CommandResult with execution details + """ + args = ["cp", f"{namespace}/{pod}:{remote_path}", str(local_path)] + if container: + args.extend(["-c", container]) + return oc(*args) + + def best_effort_oc(*oc_args: str, description: str | None = None) -> None: """Run an oc command, swallowing timeout and other errors. From 2ded63a0a04730194a9d68e1ca2f6ca74e3197b6 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 27 Jul 2026 10:35:12 +0300 Subject: [PATCH 0200/1072] [mcp_gateway] config: enable S3 historical data import/export and KPI regression analysis Add S3 backend configuration for storing and retrieving historical KPI data, enabling regression analysis against past nightly runs. Co-authored-by: Cursor --- .../mcp_gateway/orchestration/config.yaml | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index b73f7da9d..38f7c95ef 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -3,6 +3,7 @@ vaults: all-optional: - psap-forge-notifications - psap-forge-mlflow-export + - psap-forge-aws-s3-export prepare: [] test: [] @@ -34,13 +35,45 @@ caliper: enabled: false kpi: enabled: true + generate: artifacts_to_kpis: enabled: true output: kpis.json + export: + output: kpis/kpis.json + kpis_to_csv: + enabled: true + output: kpis/kpis.csv + include_header_comments: true + artifacts_to_ai_data: + enabled: false + output_dir: ai_data analyze: - enabled: false - historical_kpis: null - output: kpi_analyze.json + enabled: true + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json + + s3: + bucket: psap-dashboard-data + instance: mcp_gateway + directory: nightly + vault: + name: psap-forge-aws-s3-export + aws_credentials_file: aws.credentials + import: + enabled: true + output_dir: historical_data + include_kpis_json: true + include_kpis_csv: true + include_ai_data: false + max_downloads: 50 + export: + enabled: true + upload_id: null + dry_run: false + include_csv: true + include_kpis_json: true + include_ai_data: false export: from: null From 5ac1d5c637aacde292d9a1ea764be90362c246fc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 10:04:32 +0200 Subject: [PATCH 0201/1072] Update secret handling guidelines in AGENTS.md Clarified guidelines on handling secrets and environment variables in AGENTS.md. --- AGENTS.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ddeaaaa52..b2043d04f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,13 +40,18 @@ Key paths: 5. **Artifact directories are public.** Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. Only write non-sensitive operational data (CRs, logs, pod descriptions, events). + Pod/Deployment and other resources are safe to store, as they must not contain secrets. See point 9. + Kubernetes Secret objects should never be stored in the artifacts. Kubernetes Secret descriptions can be safely stored. -6. **Use the vault module.** +7. **Use the vault module.** Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret. -7. **Never pass secrets via environment variables** +8. **Never pass secrets via environment variables** For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care. +9. **Never pass secrets via Pod environment variables** + The Pod definition will be stored in public artifacts. Do no use plain text Pod environment variables to pass secrets to the Pod. Instead, use Secret resources and Pod environment variables that reference this secret. + ### Safe Patterns ```python From 2b61dceaaf145713d774c633ba38380f0c52b0f3 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 10:12:47 -0400 Subject: [PATCH 0202/1072] fix: resolve all ruff lint errors - Remove unused imports (csv, tempfile, runtime_config) - Replace Optional[X] with X | None (UP045) - Remove quotes from type annotation (UP037) - Use datetime.UTC alias (UP017) - Sort import blocks (I001) - Rewrite generator as set comprehension (C401) - Use dict.fromkeys instead of dict comprehension (C420) - Bind loop variable in nested function definitions (B023) Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 10 ++++------ .../toolbox/run_guidellm_benchmark/main.py | 2 +- projects/rhaiis/orchestration/analysis.py | 1 - projects/rhaiis/orchestration/test_phase.py | 9 ++++----- projects/rhaiis/postprocess/agent.py | 16 ++++++++-------- projects/rhaiis/postprocess/parser.py | 14 ++++++-------- projects/rhaiis/postprocess/plugin.py | 2 +- 7 files changed, 24 insertions(+), 30 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 847fbf159..2659446ec 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,14 +2,12 @@ from __future__ import annotations -import csv import json import logging -import tempfile import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional +from typing import Any import numpy as np @@ -28,7 +26,7 @@ class RegressionResult: is_improvement: bool = False -def geometric_mean(values) -> Optional[float]: +def geometric_mean(values) -> float | None: positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] if len(positive) == 0: return None @@ -42,7 +40,7 @@ def compare_runs( current_version: str, profile_map: dict[tuple[int, int], str], metrics: dict[str, dict[str, Any]], - restrict_profiles: Optional[list[str]] = None, + restrict_profiles: list[str] | None = None, ) -> tuple[list[RegressionResult], str]: """Compare current run metrics against a baseline version. @@ -171,7 +169,7 @@ def run_regression_analysis( output_file: Path, profile_map: dict[tuple[int, int], str], metrics: dict[str, dict[str, Any]], - restrict_profiles: Optional[list[str]] = None, + restrict_profiles: list[str] | None = None, ) -> dict[str, Any]: """Run regression analysis comparing current run against baseline. diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index e2f4309e6..befad6354 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -326,7 +326,7 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" -def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "Path | None": +def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> Path | None: """Copy a result file from a pod, compressing first to handle large files. Falls back to direct ``oc cp`` for small files. diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 5f22c36ac..71c049908 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -8,7 +8,6 @@ from pathlib import Path from projects.core.library import env -from projects.rhaiis.orchestration import runtime_config logger = logging.getLogger(__name__) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 01793a977..5a2658e57 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -117,13 +117,13 @@ def _run_test( run_uuid=run_uuid, ) + import subprocess + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready - - import subprocess fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: @@ -533,12 +533,11 @@ def _run_warmup_step( benchmark_timeout: int, ) -> None: """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" + from projects.core.library import config from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( run as run_guidellm_benchmark, ) - from projects.core.library import config - warmup_cfg = config.project.get_config("rhaiis.warmup", {}) warmup_rate = warmup_cfg.get("rate", 200) warmup_max_seconds = warmup_cfg.get("max_seconds", 60) @@ -687,7 +686,7 @@ def _upload_profiler_traces( return traces_dir = trace_files[0].parent - if len(set(f.parent for f in trace_files)) > 1: + if len({f.parent for f in trace_files}) > 1: traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces_combined" traces_dir.mkdir(parents=True, exist_ok=True) for f in trace_files: diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py index d08140310..51e5591a6 100644 --- a/projects/rhaiis/postprocess/agent.py +++ b/projects/rhaiis/postprocess/agent.py @@ -7,7 +7,7 @@ import json import logging import uuid -from typing import Optional +from datetime import UTC from urllib.error import URLError from urllib.request import Request, urlopen @@ -47,7 +47,7 @@ def _build_prompt( compare_version: str, tp: str, severe_regressions: list, - improvements: Optional[list] = None, + improvements: list | None = None, ) -> str: """Build a natural-language prompt for the agent from regression context.""" lines = [ @@ -108,8 +108,8 @@ def request_agent_analysis( severe_regressions: list, agent_url: str, job_id: str = "", - improvements: Optional[list] = None, -) -> Optional[str]: + improvements: list | None = None, +) -> str | None: """Send regression context to the PSAP agent and return its analysis. Returns the agent's response as a markdown string, or None on failure. @@ -165,7 +165,7 @@ def request_agent_analysis( return None -def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: +def send_followup(message: str, job_id: str, agent_url: str) -> str | None: """Send a followup message to the agent on an existing session. Reuses the same thread_id/session_id as request_agent_analysis so the @@ -214,7 +214,7 @@ def send_followup(message: str, job_id: str, agent_url: str) -> Optional[str]: return None -def _collect_response(resp) -> Optional[str]: +def _collect_response(resp) -> str | None: """Read the NDJSON stream and extract the final AI message content.""" collected_tokens = [] final_message = None @@ -265,9 +265,9 @@ def markdown_to_html( body = f"
{html_lib.escape(md_text)}
" - from datetime import datetime, timezone + from datetime import datetime - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC") return f""" diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 752c49297..1de53a46c 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -183,24 +183,22 @@ def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: metrics = bench.get("metrics", {}) strategy = bench.get("config", {}).get("strategy", {}) - def _pct(metric_name: str, pct: str): - return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + def _pct(metric_name: str, pct: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) def _pct_s(metric_name: str, pct: str): - """Percentile value converted from ms to seconds.""" v = _pct(metric_name, pct) return v / 1000.0 if v is not None else None - def _stat(metric_name: str, stat: str): - return metrics.get(metric_name, {}).get("successful", {}).get(stat) + def _stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get(stat) def _stat_s(metric_name: str, stat: str): - """Stat value converted from ms to seconds.""" v = _stat(metric_name, stat) return v / 1000.0 if v is not None else None - def _total_stat(metric_name: str, stat: str): - return metrics.get(metric_name, {}).get("total", {}).get(stat) + def _total_stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("total", {}).get(stat) request_totals = metrics.get("request_totals", {}) diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 658462638..bf93ae8b7 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -148,7 +148,7 @@ def export_kpis_to_csv( else f"{acc}-{model_id}-{tp}" ) - row: dict[str, Any] = {col: "" for col in FIELDNAMES} + row: dict[str, Any] = dict.fromkeys(FIELDNAMES, "") # Populate all metric columns from KPI values row.update(metrics) # Convert latency columns from seconds back to ms for dashboard compat From fd84c536dd62e408edb6963c5bcee7760b185579 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 10:24:25 -0400 Subject: [PATCH 0203/1072] style: apply ruff formatting to all changed files Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 50 ++++++---- projects/core/notifications/send.py | 1 - .../toolbox/run_guidellm_benchmark/main.py | 21 ++++- projects/rhaiis/orchestration/analysis.py | 37 ++++++-- projects/rhaiis/orchestration/ci.py | 9 +- projects/rhaiis/orchestration/test_phase.py | 68 +++++++++++--- projects/rhaiis/postprocess/csv_export.py | 92 ++++++++++++++----- projects/rhaiis/postprocess/kpis.py | 30 +++--- projects/rhaiis/postprocess/parser.py | 13 +-- projects/rhaiis/postprocess/plugin.py | 30 ++++-- projects/rhaiis/postprocess/regression.py | 32 +++++-- projects/rhaiis/postprocess/s3_dashboard.py | 7 +- .../toolbox/enable_profiler_gate/main.py | 3 +- .../toolbox/verify_profiler_prereqs/main.py | 10 +- 14 files changed, 282 insertions(+), 121 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 2659446ec..59e624f57 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -27,7 +27,9 @@ class RegressionResult: def geometric_mean(values) -> float | None: - positive = values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + positive = ( + values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + ) if len(positive) == 0: return None return float(np.exp(np.mean(np.log(positive)))) @@ -69,8 +71,14 @@ def compare_runs( accelerator = current_df["accelerator"].iloc[0] tp = current_df["TP"].iloc[0] - logger.info("Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", - current_version, compare_version, model, accelerator, tp) + logger.info( + "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, + compare_version, + model, + accelerator, + tp, + ) def _assign_profile(row): try: @@ -91,7 +99,10 @@ def _assign_profile(row): ] if baseline_filtered.empty: - return [], f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}" + return ( + [], + f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", + ) current_profiles = set(current_df["_profile"].dropna().unique()) baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) @@ -112,7 +123,9 @@ def _assign_profile(row): cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") - common_conc = set(cur["intended concurrency"].dropna().unique()) & set(base["intended concurrency"].dropna().unique()) + common_conc = set(cur["intended concurrency"].dropna().unique()) & set( + base["intended concurrency"].dropna().unique() + ) common_conc.discard(1) if not common_conc: @@ -144,20 +157,23 @@ def _assign_profile(row): is_regression = pct_diff > threshold is_improvement = pct_diff < -threshold - results.append(RegressionResult( - profile=profile, - metric_column=col, - metric_label=meta["label"], - current_value=cur_gm, - baseline_value=base_gm, - pct_diff=pct_diff, - is_regression=is_regression, - is_improvement=is_improvement, - )) + results.append( + RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + ) + ) regressions = [r for r in results if r.is_regression] - logger.info("Comparison complete: %d metric comparisons, %d regressions", - len(results), len(regressions)) + logger.info( + "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) + ) return results, "" diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 8f2d55b2e..ab6f64acc 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -714,7 +714,6 @@ def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, d return not ok - def get_slack_cpt_message(summary): def get_link(name, path, is_raw_file=False, base=None, is_dir=False): return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index befad6354..fa82ff1ec 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -339,7 +339,8 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "cp", f"{namespace}/{pod}:{remote_path}", str(local_path), - "-c", "copy-helper", + "-c", + "copy-helper", check=False, log_stdout=False, ) @@ -347,14 +348,23 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> return local_path logging.getLogger(__name__).info( - "Direct oc cp failed (rc=%d), retrying with gzip compression", result.returncode, + "Direct oc cp failed (rc=%d), retrying with gzip compression", + result.returncode, ) # Compress inside the pod, copy the gz, decompress locally gz_remote = f"{remote_path}.gz" compress = oc( - "exec", pod, "-n", namespace, "-c", "copy-helper", "--", - "gzip", "-kf", remote_path, + "exec", + pod, + "-n", + namespace, + "-c", + "copy-helper", + "--", + "gzip", + "-kf", + remote_path, check=False, ) if compress.returncode != 0: @@ -366,7 +376,8 @@ def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> "cp", f"{namespace}/{pod}:{gz_remote}", str(local_gz), - "-c", "copy-helper", + "-c", + "copy-helper", check=False, log_stdout=False, ) diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 71c049908..236e420a1 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -47,7 +47,9 @@ def run_standalone_analysis( logger.warning("AWS credentials not available, skipping standalone analysis") return - accelerator = accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + accelerator = ( + accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + ) consolidated_path = None current_csv_path = None @@ -77,7 +79,10 @@ def run_standalone_analysis( if current_rows.empty: logger.warning( "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", - version, model_id, accelerator, tp, + version, + model_id, + accelerator, + tp, ) return @@ -87,8 +92,14 @@ def run_standalone_analysis( logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) run_regression_check( - current_csv_path, compare_version, version, model_cfg, accelerator, - run_uuid=run_uuid, restrict_profiles=restrict_profiles, vllm_args=vllm_args, + current_csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid=run_uuid, + restrict_profiles=restrict_profiles, + vllm_args=vllm_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -155,8 +166,12 @@ def run_regression_check( agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) if agent_cfg.get("enabled", False): report_url = run_agent_analysis( - analysis, model_cfg, accelerator, - current_version, compare_version, run_uuid, + analysis, + model_cfg, + accelerator, + current_version, + compare_version, + run_uuid, severity_threshold=agent_cfg.get("severity_threshold", 10), ) @@ -243,7 +258,11 @@ def run_agent_analysis( agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" html_content = markdown_to_html( - agent_response, run_uuid, model, current_version, compare_version, + agent_response, + run_uuid, + model, + current_version, + compare_version, ) html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" html_path.write_text(html_content, encoding="utf-8") @@ -261,7 +280,9 @@ def run_agent_analysis( s3_bucket = s3_cfg.get("bucket", "") s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" s3.upload_file( - str(html_path), s3_bucket, s3_key, + str(html_path), + s3_bucket, + s3_key, ExtraArgs={"ContentType": "text/html"}, ) report_url = s3.generate_presigned_url( diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 763bca95c..9ba9c2e26 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -33,10 +33,7 @@ def _check_pipeline_failure_and_notify() -> None: test_ran = test_dir.exists() # Skip steps already handled by do_test's exception handler - failure_files = sorted( - f for f in base_dir.glob("*/FAILURE") - if f.parent.name != "03__test" - ) + failure_files = sorted(f for f in base_dir.glob("*/FAILURE") if f.parent.name != "03__test") if not failure_files and test_ran: return @@ -49,7 +46,9 @@ def _check_pipeline_failure_and_notify() -> None: errors.append(f"[{step_name}] {summary}") if not test_ran and not errors: - errors.append("Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)") + errors.append( + "Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)" + ) error_text = "\n".join(errors) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 5a2658e57..ceb138947 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -14,7 +14,6 @@ _warnings: list[str] = [] - def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -63,6 +62,7 @@ def do_test( ) except Exception as exc: from projects.rhaiis.orchestration.notifications import send_pipeline_failure_alert + send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) raise @@ -78,6 +78,7 @@ def _run_test( accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator from projects.core.library import config as _cfg + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() deploy_cfg = runtime_config.get_deploy_config() @@ -103,12 +104,16 @@ def _run_test( env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) from projects.core.library import config as _cfg + version = _cfg.project.get_config("tests.rhaiis.version", "") run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) _create_test_labels( - model_key, workload_keys[0], accelerator, vllm_args, + model_key, + workload_keys[0], + accelerator, + vllm_args, hf_model_id=model_cfg["hf_model_id"], version=version, vllm_image=vllm_image, @@ -124,15 +129,27 @@ def _run_test( ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready + fjob_name = os.environ.get("FJOB_NAME", "") fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") if fjob_name: try: mgmt_env = {k: v for k, v in os.environ.items() if k != "KUBECONFIG"} subprocess.run( - ["oc", "annotate", "fournosjob", fjob_name, "-n", fjob_ns, - f"rhaiis.run-uuid={run_uuid}", "--overwrite"], - check=False, capture_output=True, timeout=10, env=mgmt_env, + [ + "oc", + "annotate", + "fournosjob", + fjob_name, + "-n", + fjob_ns, + f"rhaiis.run-uuid={run_uuid}", + "--overwrite", + ], + check=False, + capture_output=True, + timeout=10, + env=mgmt_env, ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: @@ -149,9 +166,13 @@ def _run_test( logger.info("run_benchmark=false and profiler=false, running standalone analysis only") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis( - model_cfg, accelerator_key, vllm_args, - run_uuid=run_uuid, restrict_profiles=workload_keys or None, + model_cfg, + accelerator_key, + vllm_args, + run_uuid=run_uuid, + restrict_profiles=workload_keys or None, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -202,7 +223,9 @@ def _run_test( ) logger.info( - "Running %d workload(s): %s", len(workload_keys), workload_keys, + "Running %d workload(s): %s", + len(workload_keys), + workload_keys, ) # Phase 1: warmup or profiler for ALL workloads first @@ -287,8 +310,11 @@ def _run_test( _warnings.append("Predictor log upload failed") if _warnings: - logger.warning("Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings)) + logger.warning( + "Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings) + ) from projects.rhaiis.orchestration.notifications import send_pipeline_warning + send_pipeline_warning( warnings=list(_warnings), model_key=model_key, @@ -335,6 +361,7 @@ def _run_workload_benchmark( logger.info("run_benchmark=false, skipping main benchmark") try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -456,7 +483,9 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) logger.info("No postprocessing directory found, skipping dashboard CSV sync") return - kpis_to_csv_output = config.project.get_config("caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv") + kpis_to_csv_output = config.project.get_config( + "caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv" + ) csv_path = postprocess_dirs[-1] / kpis_to_csv_output if not csv_path.exists(): logger.info("Dashboard CSV not found at %s, skipping sync", csv_path) @@ -489,9 +518,15 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) from projects.rhaiis.orchestration.analysis import run_regression_check + run_regression_check( - csv_path, compare_version, version, model_cfg, accelerator, - run_uuid="", vllm_args=vllm_args, + csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid="", + vllm_args=vllm_args, ) @@ -502,7 +537,9 @@ def _upload_predictor_log(run_uuid: str) -> None: from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 - matches = sorted(Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs")) + matches = sorted( + Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs") + ) log_path = matches[-1] if matches else None if not log_path or not log_path.exists(): logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) @@ -680,7 +717,9 @@ def _upload_profiler_traces( from projects.core.library import config from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 - trace_files = sorted(Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*")) + trace_files = sorted( + Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*") + ) if not trace_files: logger.info("No profiler traces to upload") return @@ -691,6 +730,7 @@ def _upload_profiler_traces( traces_dir.mkdir(parents=True, exist_ok=True) for f in trace_files: import shutil + shutil.copy2(f, traces_dir / f.name) logger.info("Found %d profiler trace files in %s", len(trace_files), traces_dir) diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py index 93270ac27..5f86396dc 100644 --- a/projects/rhaiis/postprocess/csv_export.py +++ b/projects/rhaiis/postprocess/csv_export.py @@ -14,19 +14,58 @@ logger = logging.getLogger(__name__) FIELDNAMES = [ - "run", "accelerator", "model", "version", "prompt toks", "output toks", "TP", - "measured concurrency", "intended concurrency", "measured rps", - "output_tok/sec", "total_tok/sec", "prompt_token_count_mean", - "prompt_token_count_p99", "output_token_count_mean", - "output_token_count_p99", "ttft_median", "ttft_p95", "ttft_p1", "ttft_p999", - "tpot_median", "tpot_p95", "tpot_p99", "tpot_p999", "tpot_p1", - "itl_median", "itl_p95", "itl_p999", "itl_p1", - "request_latency_median", "request_latency_min", "request_latency_max", - "successful_requests", "errored_requests", "uuid", - "ttft_mean", "ttft_p99", "itl_mean", "itl_p99", "runtime_args", - "guidellm_start_time_ms", "guidellm_end_time_ms", "image_tag", "guidellm_version", - "DP", "dataset", "spec_decoding", "prefix_caching", - "turns", "prefix_tokens", "prefix_count", "request_type", + "run", + "accelerator", + "model", + "version", + "prompt toks", + "output toks", + "TP", + "measured concurrency", + "intended concurrency", + "measured rps", + "output_tok/sec", + "total_tok/sec", + "prompt_token_count_mean", + "prompt_token_count_p99", + "output_token_count_mean", + "output_token_count_p99", + "ttft_median", + "ttft_p95", + "ttft_p1", + "ttft_p999", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p999", + "tpot_p1", + "itl_median", + "itl_p95", + "itl_p999", + "itl_p1", + "request_latency_median", + "request_latency_min", + "request_latency_max", + "successful_requests", + "errored_requests", + "uuid", + "ttft_mean", + "ttft_p99", + "itl_mean", + "itl_p99", + "runtime_args", + "guidellm_start_time_ms", + "guidellm_end_time_ms", + "image_tag", + "guidellm_version", + "DP", + "dataset", + "spec_decoding", + "prefix_caching", + "turns", + "prefix_tokens", + "prefix_count", + "request_type", ] @@ -51,11 +90,15 @@ def _format_runtime_args(inference_server_args: dict, trtllm_config: dict | None parts = [] for key, value in (inference_server_args or {}).items(): formatted_key = key.replace("_", "-") - formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) parts.append(f"{formatted_key}: {formatted_value}") for key, value in (trtllm_config or {}).items(): formatted_key = f"trtllm.{key.replace('_', '-')}" - formatted_value = json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) parts.append(f"{formatted_key}: {formatted_value}") return "; ".join(parts) @@ -65,7 +108,9 @@ def find_psap_files(artifact_dir: Path) -> list[Path]: return sorted(artifact_dir.rglob("PSAP_perf*.json")) -def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "") -> Path: +def generate_dashboard_csv( + psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "" +) -> Path: """Read a PSAP JSON file and produce a dashboard CSV.""" with open(psap_json_path, encoding="utf-8") as f: payload = json.load(f) @@ -108,8 +153,12 @@ def generate_dashboard_csv(psap_json_path: Path, version: str, output_path: Path start_times.append(sched["start_time"]) if "end_time" in sched: end_times.append(sched["end_time"]) - guidellm_start_ms = int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") - guidellm_end_ms = int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") + guidellm_start_ms = ( + int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") + ) + guidellm_end_ms = ( + int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") + ) rows = [] for bench in benchmarks: @@ -165,12 +214,7 @@ def _extract_row( run_uuid: str = "", ) -> dict: def _pct(metric_name: str, pct: str): - return ( - metrics.get(metric_name, {}) - .get("successful", {}) - .get("percentiles", {}) - .get(pct) - ) + return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) def _stat(metric_name: str, stat: str): return metrics.get(metric_name, {}).get("successful", {}).get(stat) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index a262bb41d..fc137feea 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -110,19 +110,21 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if higher_is_better is not None: labels["higher_is_better"] = higher_is_better - out.append({ - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_id": r.test_base_path, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": model.plugin_module, - }, - }) + out.append( + { + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_id": r.test_base_path, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": model.plugin_module, + }, + } + ) return out diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 1de53a46c..a607811f8 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -68,10 +68,7 @@ def _find_node_for_record( def _bench_sort_key(bench: dict) -> float: """Sort key matching GuideLLMParser: requests_per_second.successful.mean.""" return float( - bench.get("metrics", {}) - .get("requests_per_second", {}) - .get("successful", {}) - .get("mean", 0) + bench.get("metrics", {}).get("requests_per_second", {}).get("successful", {}).get("mean", 0) ) @@ -79,8 +76,12 @@ def _extract_extra_metrics(node: TestBaseNode) -> tuple[dict[str, Any], dict[str """Return (scalar_metrics, extra_curves) extracted from raw benchmarks.json files.""" extra: dict[str, Any] = {} benchmarks_files = sorted( - [p for p in node.artifact_paths if p.name == "benchmarks.json" - or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json")], + [ + p + for p in node.artifact_paths + if p.name == "benchmarks.json" + or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json") + ], key=lambda p: p.name, ) if not benchmarks_files: diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index bf93ae8b7..3c6765053 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -20,11 +20,27 @@ # CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. # GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. -_SECONDS_TO_MS_COLUMNS = frozenset({ - "ttft_median", "ttft_p95", "ttft_p99", "ttft_p1", "ttft_p999", "ttft_mean", - "tpot_median", "tpot_p95", "tpot_p99", "tpot_p1", "tpot_p999", - "itl_median", "itl_p95", "itl_p99", "itl_p1", "itl_p999", "itl_mean", -}) +_SECONDS_TO_MS_COLUMNS = frozenset( + { + "ttft_median", + "ttft_p95", + "ttft_p99", + "ttft_p1", + "ttft_p999", + "ttft_mean", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p1", + "tpot_p999", + "itl_median", + "itl_p95", + "itl_p99", + "itl_p1", + "itl_p999", + "itl_mean", + } +) # KPI ID → dashboard CSV column name mapping (complete) _KPI_TO_CSV_COLUMN = { @@ -143,9 +159,7 @@ def export_kpis_to_csv( version = labels.get("version", "") run_name = ( - f"{acc}-{cluster_tag}-{model_id}-{tp}" - if cluster_tag - else f"{acc}-{model_id}-{tp}" + f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" ) row: dict[str, Any] = dict.fromkeys(FIELDNAMES, "") diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 1b524f7fa..6187a52f1 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -14,7 +14,9 @@ RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" -def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None, channel_id: str | None = None) -> bool: +def _send_via_topsail_bot( + message: str, *, notification_vault: str | None = None, channel_id: str | None = None +) -> bool: """Send a Slack message using the topsail bot token from the forge notifications vault.""" vault_name = notification_vault or "psap-forge-notifications" try: @@ -49,7 +51,11 @@ def _send_via_topsail_bot(message: str, *, notification_vault: str | None = None "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, - "request_latency_median": {"label": "Median E2E Latency", "higher_is_better": False, "threshold": 5}, + "request_latency_median": { + "label": "Median E2E Latency", + "higher_is_better": False, + "threshold": 5, + }, } DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" @@ -161,10 +167,16 @@ def send_regression_notification( detail_lines = [] all_results = analysis_result.get("all_results", []) - profiles = sorted({r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")}) + profiles = sorted( + {r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")} + ) for profile in profiles: - profile_results = [r for r in all_results if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement"))] + profile_results = [ + r + for r in all_results + if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement")) + ] detail_lines.append(f"\n*{profile}:*") for r in profile_results: if r.get("is_regression"): @@ -226,7 +238,9 @@ def send_regression_notification( logger.info("DRY RUN regression notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) def send_failure_notification( @@ -275,7 +289,9 @@ def send_failure_notification( parallelism_parts.append(f"TP={tp}") if dp: parallelism_parts.append(f"DP={dp}") - parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + parallelism_line = ( + f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + ) profiles_line = "" if workload_keys: @@ -303,4 +319,6 @@ def send_failure_notification( logger.info("DRY RUN failure notification:\n%s", message) return True - return _send_via_topsail_bot(message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID) + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py index c648ce9e7..060288284 100644 --- a/projects/rhaiis/postprocess/s3_dashboard.py +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -72,6 +72,7 @@ def sync_csv_to_s3( return {"status": "failed", "error": str(e)} finally: import os + if consolidated_path and os.path.exists(consolidated_path): os.unlink(consolidated_path) @@ -98,9 +99,9 @@ def upload_profiler_traces_to_s3( return {"status": "skipped", "reason": f"traces dir not found: {traces_dir}"} trace_files = [ - f for f in traces_dir.iterdir() - if f.name.startswith("trace") and "rank0" in f.name - and f.suffix in (".json", ".gz") + f + for f in traces_dir.iterdir() + if f.name.startswith("trace") and "rank0" in f.name and f.suffix in (".json", ".gz") ] if not trace_files: return {"status": "skipped", "reason": "no trace files found"} diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py index b1783b09c..c370953ed 100644 --- a/projects/rhaiis/toolbox/enable_profiler_gate/main.py +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -44,8 +44,7 @@ def find_predictor_pod(args, context): def toggle_profiler_gate(args, context): if args.disable: shell.run( - f"oc exec {context.pod_name} -n {args.namespace} " - "-- sh -c 'rm -f /tmp/profiler_gate'", + f"oc exec {context.pod_name} -n {args.namespace} -- sh -c 'rm -f /tmp/profiler_gate'", check=False, ) return f"Profiler gate disabled on {context.pod_name}" diff --git a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py index dc2e95e50..2fd23542d 100644 --- a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py +++ b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py @@ -24,8 +24,7 @@ def run(*, namespace: str): @task def check_webhook_pod(args, context): result = shell.run( - "oc get pods -n vllm-profiler -l app=env-injector " - "-o jsonpath='{.items[0].status.phase}'", + "oc get pods -n vllm-profiler -l app=env-injector -o jsonpath='{.items[0].status.phase}'", check=False, ) phase = result.stdout.strip().strip("'") @@ -99,8 +98,7 @@ def check_tls_secret(args, context): @task def check_configmap(args, context): result = shell.run( - f"oc get configmap env-injector-files -n {args.namespace} " - "-o jsonpath='{.data}'", + f"oc get configmap env-injector-files -n {args.namespace} -o jsonpath='{{.data}}'", check=False, ) if result.returncode != 0 or "sitecustomize.py" not in result.stdout: @@ -117,9 +115,7 @@ def check_configmap(args, context): def raise_on_errors(args, context): errors = getattr(context, "errors", []) if errors: - raise ValueError( - "Profiler prerequisites not met:\n - " + "\n - ".join(errors) - ) + raise ValueError("Profiler prerequisites not met:\n - " + "\n - ".join(errors)) return "All profiler prerequisites verified" From 61fdcd1808e94480e789ae23a5ab92776f852499 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 11:59:30 -0400 Subject: [PATCH 0204/1072] refactor: move CSV regression analysis from caliper engine to rhaiis plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move compare_runs, run_regression_analysis, RegressionResult, and geometric_mean out of the generic caliper/engine/kpi/analyze.py into rhaiis/postprocess/regression.py. The caliper engine should contain no project-specific code (no references to model, accelerator, TP, tokens, etc.). This is a temporary transition — these CSV-based comparison functions will be replaced by generic Caliper KPI-based analysis once the framework lands. Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 257 +------------------- projects/rhaiis/orchestration/analysis.py | 3 +- projects/rhaiis/postprocess/regression.py | 273 +++++++++++++++++++++- 3 files changed, 274 insertions(+), 259 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 59e624f57..47ae439ff 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -1,271 +1,16 @@ -"""Regression analysis vs baseline KPI set.""" +"""Generic KPI analysis against historical baselines.""" from __future__ import annotations import json import logging import time -from dataclasses import dataclass from pathlib import Path from typing import Any -import numpy as np - logger = logging.getLogger(__name__) -@dataclass -class RegressionResult: - profile: str - metric_column: str - metric_label: str - current_value: float - baseline_value: float - pct_diff: float - is_regression: bool - is_improvement: bool = False - - -def geometric_mean(values) -> float | None: - positive = ( - values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] - ) - if len(positive) == 0: - return None - return float(np.exp(np.mean(np.log(positive)))) - - -def compare_runs( - current_csv_path: str, - consolidated_csv_path: str, - compare_version: str, - current_version: str, - profile_map: dict[tuple[int, int], str], - metrics: dict[str, dict[str, Any]], - restrict_profiles: list[str] | None = None, -) -> tuple[list[RegressionResult], str]: - """Compare current run metrics against a baseline version. - - Args: - profile_map: Mapping of (input_tokens, output_tokens) -> profile name. - metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. - - Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. - """ - import pandas as pd - - current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") - baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") - - for col in current_df.columns: - if current_df[col].dtype == object: - current_df[col] = current_df[col].str.strip() - for col in baseline_df.columns: - if baseline_df[col].dtype == object: - baseline_df[col] = baseline_df[col].str.strip() - - if current_df.empty: - return [], "Current run produced no benchmark data" - - model = current_df["model"].iloc[0] - accelerator = current_df["accelerator"].iloc[0] - tp = current_df["TP"].iloc[0] - - logger.info( - "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", - current_version, - compare_version, - model, - accelerator, - tp, - ) - - def _assign_profile(row): - try: - isl = int(float(row["prompt toks"])) - osl = int(float(row["output toks"])) - except (ValueError, TypeError): - return None - return profile_map.get((isl, osl)) - - current_df["_profile"] = current_df.apply(_assign_profile, axis=1) - baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) - - baseline_filtered = baseline_df[ - (baseline_df["version"] == compare_version) - & (baseline_df["model"] == model) - & (baseline_df["accelerator"] == accelerator) - & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) - ] - - if baseline_filtered.empty: - return ( - [], - f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", - ) - - current_profiles = set(current_df["_profile"].dropna().unique()) - baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) - common_profiles = sorted(current_profiles & baseline_profiles) - - if restrict_profiles: - common_profiles = [p for p in common_profiles if p in restrict_profiles] - - if not common_profiles: - return [], "No common ISL/OSL profiles between current and baseline" - - results: list[RegressionResult] = [] - - for profile in common_profiles: - cur = current_df[current_df["_profile"] == profile].copy() - base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() - - cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") - base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") - - common_conc = set(cur["intended concurrency"].dropna().unique()) & set( - base["intended concurrency"].dropna().unique() - ) - common_conc.discard(1) - - if not common_conc: - continue - - cur_common = cur[cur["intended concurrency"].isin(common_conc)] - base_common = base[base["intended concurrency"].isin(common_conc)] - - for col, meta in metrics.items(): - if col not in cur_common.columns or col not in base_common.columns: - continue - - cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() - base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() - - cur_gm = geometric_mean(cur_vals) - base_gm = geometric_mean(base_vals) - - if cur_gm is None or base_gm is None: - continue - - pct_diff = ((cur_gm - base_gm) / base_gm) * 100 - threshold = meta["threshold"] - - if meta["higher_is_better"]: - is_regression = pct_diff < -threshold - is_improvement = pct_diff > threshold - else: - is_regression = pct_diff > threshold - is_improvement = pct_diff < -threshold - - results.append( - RegressionResult( - profile=profile, - metric_column=col, - metric_label=meta["label"], - current_value=cur_gm, - baseline_value=base_gm, - pct_diff=pct_diff, - is_regression=is_regression, - is_improvement=is_improvement, - ) - ) - - regressions = [r for r in results if r.is_regression] - logger.info( - "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) - ) - return results, "" - - -def run_regression_analysis( - current_csv_path: Path, - consolidated_csv_path: Path, - compare_version: str, - current_version: str, - output_file: Path, - profile_map: dict[tuple[int, int], str], - metrics: dict[str, dict[str, Any]], - restrict_profiles: list[str] | None = None, -) -> dict[str, Any]: - """Run regression analysis comparing current run against baseline. - - Returns dict with status, regressions, improvements, and skip_reason. - """ - try: - results, skip_reason = compare_runs( - str(current_csv_path), - str(consolidated_csv_path), - compare_version, - current_version, - profile_map=profile_map, - metrics=metrics, - restrict_profiles=restrict_profiles, - ) - - if skip_reason: - analysis = { - "status": "skipped", - "reason": skip_reason, - "current_version": current_version, - "compare_version": compare_version, - } - else: - regressions = [r for r in results if r.is_regression] - improvements = [r for r in results if r.is_improvement] - - analysis = { - "status": "completed", - "current_version": current_version, - "compare_version": compare_version, - "total_comparisons": len(results), - "regression_count": len(regressions), - "improvement_count": len(improvements), - "regressions": [ - { - "profile": r.profile, - "metric": r.metric_label, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - } - for r in regressions - ], - "improvements": [ - { - "profile": r.profile, - "metric": r.metric_label, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - } - for r in improvements - ], - "all_results": [ - { - "profile": r.profile, - "metric": r.metric_label, - "metric_column": r.metric_column, - "current": round(r.current_value, 2), - "baseline": round(r.baseline_value, 2), - "pct_diff": round(r.pct_diff, 1), - "is_regression": r.is_regression, - "is_improvement": r.is_improvement, - } - for r in results - ], - } - - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis, f, indent=2) - - return analysis - - except Exception as e: - logger.exception("Regression analysis failed") - return {"status": "failed", "error": str(e)} - - def run_kpi_analysis( current_kpi_file: Path, historical_data_dir: Path, diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 236e420a1..f41390aab 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -122,9 +122,8 @@ def run_regression_check( vllm_args: dict | None = None, ) -> None: from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials - from projects.caliper.engine.kpi.analyze import run_regression_analysis from projects.core.library import config - from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP, run_regression_analysis s3_cfg = config.project.get_config("rhaiis.s3", {}) csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 6187a52f1..b71bec30d 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -1,9 +1,19 @@ -"""RHAIIS-specific profile/metric definitions and Slack notification for regression analysis.""" +"""RHAIIS-specific regression analysis and Slack notifications. + +Temporary home for CSV-based regression comparison ported from model-furnace. +Will be replaced by generic Caliper KPI-based analysis once the framework lands. +""" from __future__ import annotations +import json import logging import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np import projects.core.notifications.slack.api as slack_api from projects.core.library import vault @@ -39,6 +49,267 @@ def _send_via_topsail_bot( return ok +# --------------------------------------------------------------------------- +# CSV-based regression comparison (ported from model-furnace, temporary) +# --------------------------------------------------------------------------- + + +@dataclass +class RegressionResult: + profile: str + metric_column: str + metric_label: str + current_value: float + baseline_value: float + pct_diff: float + is_regression: bool + is_improvement: bool = False + + +def geometric_mean(values) -> float | None: + positive = ( + values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + ) + if len(positive) == 0: + return None + return float(np.exp(np.mean(np.log(positive)))) + + +def compare_runs( + current_csv_path: str, + consolidated_csv_path: str, + compare_version: str, + current_version: str, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> tuple[list[RegressionResult], str]: + """Compare current run metrics against a baseline version. + + Args: + profile_map: Mapping of (input_tokens, output_tokens) -> profile name. + metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. + + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. + """ + import pandas as pd + + current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") + baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") + + for col in current_df.columns: + if current_df[col].dtype == object: + current_df[col] = current_df[col].str.strip() + for col in baseline_df.columns: + if baseline_df[col].dtype == object: + baseline_df[col] = baseline_df[col].str.strip() + + if current_df.empty: + return [], "Current run produced no benchmark data" + + model = current_df["model"].iloc[0] + accelerator = current_df["accelerator"].iloc[0] + tp = current_df["TP"].iloc[0] + + logger.info( + "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, + compare_version, + model, + accelerator, + tp, + ) + + def _assign_profile(row): + try: + isl = int(float(row["prompt toks"])) + osl = int(float(row["output toks"])) + except (ValueError, TypeError): + return None + return profile_map.get((isl, osl)) + + current_df["_profile"] = current_df.apply(_assign_profile, axis=1) + baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) + + baseline_filtered = baseline_df[ + (baseline_df["version"] == compare_version) + & (baseline_df["model"] == model) + & (baseline_df["accelerator"] == accelerator) + & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) + ] + + if baseline_filtered.empty: + return ( + [], + f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", + ) + + current_profiles = set(current_df["_profile"].dropna().unique()) + baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) + common_profiles = sorted(current_profiles & baseline_profiles) + + if restrict_profiles: + common_profiles = [p for p in common_profiles if p in restrict_profiles] + + if not common_profiles: + return [], "No common ISL/OSL profiles between current and baseline" + + results: list[RegressionResult] = [] + + for profile in common_profiles: + cur = current_df[current_df["_profile"] == profile].copy() + base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() + + cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") + base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") + + common_conc = set(cur["intended concurrency"].dropna().unique()) & set( + base["intended concurrency"].dropna().unique() + ) + common_conc.discard(1) + + if not common_conc: + continue + + cur_common = cur[cur["intended concurrency"].isin(common_conc)] + base_common = base[base["intended concurrency"].isin(common_conc)] + + for col, meta in metrics.items(): + if col not in cur_common.columns or col not in base_common.columns: + continue + + cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() + base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() + + cur_gm = geometric_mean(cur_vals) + base_gm = geometric_mean(base_vals) + + if cur_gm is None or base_gm is None: + continue + + pct_diff = ((cur_gm - base_gm) / base_gm) * 100 + threshold = meta["threshold"] + + if meta["higher_is_better"]: + is_regression = pct_diff < -threshold + is_improvement = pct_diff > threshold + else: + is_regression = pct_diff > threshold + is_improvement = pct_diff < -threshold + + results.append( + RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + ) + ) + + regressions = [r for r in results if r.is_regression] + logger.info( + "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) + ) + return results, "" + + +def run_regression_analysis( + current_csv_path: Path, + consolidated_csv_path: Path, + compare_version: str, + current_version: str, + output_file: Path, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> dict[str, Any]: + """Run regression analysis comparing current run against baseline. + + Returns dict with status, regressions, improvements, and skip_reason. + """ + try: + results, skip_reason = compare_runs( + str(current_csv_path), + str(consolidated_csv_path), + compare_version, + current_version, + profile_map=profile_map, + metrics=metrics, + restrict_profiles=restrict_profiles, + ) + + if skip_reason: + analysis = { + "status": "skipped", + "reason": skip_reason, + "current_version": current_version, + "compare_version": compare_version, + } + else: + regressions = [r for r in results if r.is_regression] + improvements = [r for r in results if r.is_improvement] + + analysis = { + "status": "completed", + "current_version": current_version, + "compare_version": compare_version, + "total_comparisons": len(results), + "regression_count": len(regressions), + "improvement_count": len(improvements), + "regressions": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in regressions + ], + "improvements": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in improvements + ], + "all_results": [ + { + "profile": r.profile, + "metric": r.metric_label, + "metric_column": r.metric_column, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + "is_regression": r.is_regression, + "is_improvement": r.is_improvement, + } + for r in results + ], + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis, f, indent=2) + + return analysis + + except Exception as e: + logger.exception("Regression analysis failed") + return {"status": "failed", "error": str(e)} + + +# --------------------------------------------------------------------------- +# RHAIIS constants +# --------------------------------------------------------------------------- + PROFILE_MAP = { (1000, 1000): "profile1", (512, 2048): "profile2", From 5668caf0cccafe700e3c56bde240c3c07453415f Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 27 Jul 2026 13:58:12 -0400 Subject: [PATCH 0205/1072] fix: separate test nodes per workload for correct dashboard CSV per-profile data Previously, all workload benchmarks ran under a single NextArtifactDir ("testing") with one set of test labels (workload_key=profile1). The Caliper parser merged all benchmarks.json files into one record, causing: - Dashboard showing only "profile1" regardless of which profiles ran - Multiple data points on the same chart from different profiles Now each workload's benchmark phase runs in its own NextArtifactDir (e.g., "benchmark_profile1", "benchmark_profile2") with per-workload test labels. The parser creates separate records per profile, producing correct per-profile CSV rows and dashboard entries. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 95 +++++++++++---------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index ceb138947..f94812476 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -109,19 +109,6 @@ def _run_test( run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) logger.info("Run UUID for this job: %s", run_uuid) - _create_test_labels( - model_key, - workload_keys[0], - accelerator, - vllm_args, - hf_model_id=model_cfg["hf_model_id"], - version=version, - vllm_image=vllm_image, - cluster_tag=cluster_tag, - accelerator_chip=gpu_type.upper(), - run_uuid=run_uuid, - ) - import subprocess from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -279,6 +266,8 @@ def _run_test( endpoint_url=endpoint_url, benchmark_timeout=benchmark_timeout, run_uuid=run_uuid, + version=version, + cluster_tag=cluster_tag, ) try: @@ -342,8 +331,15 @@ def _run_workload_benchmark( endpoint_url: str, benchmark_timeout: int, run_uuid: str, + version: str, + cluster_tag: str, ) -> None: - """Run benchmark and post-processing for a single workload.""" + """Run benchmark and post-processing for a single workload. + + Each workload gets its own NextArtifactDir with per-workload test labels so + the Caliper parser treats them as separate test nodes and produces one + dashboard CSV row-set per profile. + """ logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) workload = runtime_config.get_workload(workload_key) @@ -357,41 +353,52 @@ def _run_workload_benchmark( run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) - if not run_benchmark: - logger.info("run_benchmark=false, skipping main benchmark") - try: - from projects.rhaiis.orchestration.analysis import run_standalone_analysis + with env.NextArtifactDir(f"benchmark_{workload_key}"): + _create_test_labels( + model_key, + workload_key, + accelerator, + vllm_args, + hf_model_id=model_cfg["hf_model_id"], + version=version, + vllm_image=vllm_image, + cluster_tag=cluster_tag, + accelerator_chip=gpu_type.upper(), + run_uuid=run_uuid, + ) - run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) - except Exception: - logger.warning("Standalone analysis failed", exc_info=True) - _warnings.append(f"Standalone analysis failed for {workload_key}") - else: - logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + try: + from projects.rhaiis.orchestration.analysis import run_standalone_analysis - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append(f"Standalone analysis failed for {workload_key}") + else: + logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, - ) + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, - hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), - ) + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) - # Dashboard CSV generation and S3 sync is handled by the Caliper - # postprocessing pipeline (kpis_to_csv step) + _sync_postprocessed_dashboard_csv + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + ) def _create_test_labels( From d57c523f16c9ef9d30000065f26d1256007d4e24 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:34:02 +0200 Subject: [PATCH 0206/1072] [core] dsl: task: log the end time --- projects/core/dsl/task.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/projects/core/dsl/task.py b/projects/core/dsl/task.py index 40e31017c..bd593a512 100644 --- a/projects/core/dsl/task.py +++ b/projects/core/dsl/task.py @@ -59,6 +59,7 @@ def _log_retry_attempt( result=None, exc=None, artifact_dirname_suffix=None, + retry_backoff=1.0, ): """ Log a retry attempt with consistent formatting. @@ -115,7 +116,21 @@ def _log_retry_attempt( retry_reason = f"returned: {result}" logger.warning(f"~~ RETRY ATTEMPT #{attempt + 1}/{retry_attempts} ({retry_reason})") - logger.info(f"~~ RETRY in {current_delay:.0f}s") + + # Calculate total remaining time for all future retries with backoff + remaining_attempts = retry_attempts - 1 - attempt + total_remaining_time = 0 + delay_for_calculation = current_delay + + for _i in range(remaining_attempts): + total_remaining_time += delay_for_calculation + delay_for_calculation *= retry_backoff + + # Calculate when retries will end + end_time = time.time() + total_remaining_time + end_timestamp = datetime.fromtimestamp(end_time).strftime("%H:%M") + + logger.info(f"~~ RETRY in {current_delay:.0f}s until {end_timestamp}") logger.info("~" * LINE_WIDTH) logger.info("") @@ -187,6 +202,7 @@ def _execute_with_retry( retry_reason=retry_reason, result=result, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff else: @@ -239,6 +255,7 @@ def _execute_with_retry( current_delay, exc=exc, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff From 407ca5012b95ecaab153878a9c24afb0ff53534d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:03:59 +0200 Subject: [PATCH 0207/1072] [core] dsl: runtime: reduce the log level --- projects/core/dsl/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index 78ff145d6..c0c4b8b88 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -674,7 +674,7 @@ def _generate_failure_file_for_agent(exception: Exception) -> None: with open(failure_file_path, "w", encoding="utf-8") as f: f.write(failure_content) - logger.info(f"Generated FAILURE file for agent analysis: {failure_file_path}") + logger.debug(f"Generated FAILURE file for agent analysis: {failure_file_path}") except Exception as failure_error: logger.warning(f"Failed to generate FAILURE file for agent: {failure_error}") From b3f7964c5059d142c4ab6633f58a0ce26a69c86b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:09:30 +0200 Subject: [PATCH 0208/1072] [core] library: export: improve the test status --- projects/core/library/export.py | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index c1eff68f1..a115a84c3 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -729,6 +729,12 @@ def _build_enhanced_notification( notification_parts.append("**Test Logs**") notification_parts.extend(step_log_links) + # Add distinct test and post-processing status right under Test Logs + test_status_section = _build_test_status_section(status) + if test_status_section: + notification_parts.append("") + notification_parts.extend(test_status_section) + if postprocess_status_links: notification_parts.append("") notification_parts.extend(postprocess_status_links) @@ -740,6 +746,42 @@ def _build_enhanced_notification( return "\n".join(notification_parts) +def _build_test_status_section(status: dict[str, Any]) -> list[str]: + """Build distinct test and post-processing status section.""" + try: + test_phase = status.get("test_phase", {}) + if not test_phase: + return [] + + test_status = test_phase.get("phase", "UNKNOWN") + test_message = test_phase.get("message", "") + + # Determine post-processing status based on final status and test outcome + final_status = status.get("final_status", "unknown") + if test_status == "FAILED": + post_processing_status = "skipped" # Don't run post-processing if test failed + elif final_status == "success": + post_processing_status = "success" + elif "failed" in final_status.lower(): + post_processing_status = "failed" + else: + post_processing_status = "unknown" + + status_lines = [f"**test:** {test_status}"] + + if test_message: + # Format message with blockquote-style prefix + status_lines.append(f"> {test_message}") + + status_lines.append(f"**post-processing:** {post_processing_status}") + + return status_lines + + except Exception as e: + logger.warning(f"Failed to build test status section: {e}") + return [] + + def _extract_project_from_status(status: dict[str, Any]) -> str: """Extract project name from status object or environment.""" # Try to get project from environment variables From 7635cda01d981fdebbb4eb16a456119040ce7c92 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:45:24 +0200 Subject: [PATCH 0209/1072] [core] library: config: write_variables_override: accept comma separated presets --- projects/core/library/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index 9dd5d51f2..0d1fa1303 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -558,6 +558,9 @@ def write_variables_override(presets=None, variables_dict=None): # Set presets in project.args if presets: + if len(presets) == 1 and "," in presets[0]: + presets = list(presets[0].split(",")) + overrides["project.args"] = presets logger.info(f"write_variables_override: project.args --> {presets}") From 0c9ce6066d2b8d438639dab4185a4ee30b442aca Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:38:37 +0200 Subject: [PATCH 0210/1072] [core] library: env: add a MuteStdOut context --- projects/core/library/env.py | 78 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 4def14e1e..e2ef4bbb2 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -3,6 +3,8 @@ import pathlib import threading import time +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO # ARTIFACT_DIR is accessed via __getattr__ for thread-local support BASE_ARTIFACT_DIR = None # Immutable copy of the initial ARTIFACT_DIR @@ -33,7 +35,7 @@ def __getattr__(name): ) return globals().get("_GLOBAL_ARTIFACT_DIR") - return globals()[name] + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") def get_tls_artifact_dir(): @@ -215,3 +217,77 @@ def next_artifact_index(): return 0 return len(list(current_artifact_dir.glob("*__*"))) + + +class MuteStdOut: + """Context manager to mute stdout, stderr, and logging with a startup message.""" + + def __init__(self, reason: str): + self.reason = reason + self.captured_stdout = StringIO() + self.captured_stderr = StringIO() + self.original_logging_level = None + self.original_handlers_streams = [] + + def __enter__(self): + # Print the startup message before muting + logger = logging.getLogger(__name__) + logger.info(self.reason) + + # Redirect logging handlers that have direct stream references + self._redirect_logging_handlers() + + # Suppress logging by setting root logger level to CRITICAL + root_logger = logging.getLogger() + self.original_logging_level = root_logger.level + root_logger.setLevel(logging.CRITICAL) + + # Start capturing stdout and stderr + self.stdout_redirect = redirect_stdout(self.captured_stdout) + self.stderr_redirect = redirect_stderr(self.captured_stderr) + + self.stdout_redirect.__enter__() + self.stderr_redirect.__enter__() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Restore stdout and stderr + self.stdout_redirect.__exit__(exc_type, exc_val, exc_tb) + self.stderr_redirect.__exit__(exc_type, exc_val, exc_tb) + + # Restore original logging level + if self.original_logging_level is not None: + root_logger = logging.getLogger() + root_logger.setLevel(self.original_logging_level) + + # Restore original logging handler streams + self._restore_logging_handlers() + + return False + + def _redirect_logging_handlers(self): + """Redirect logging handlers with direct stream references to captured streams.""" + import sys + + # Save and redirect handlers from root logger and all existing loggers + all_loggers = [logging.getLogger()] + [ + logging.getLogger(name) for name in logging.Logger.manager.loggerDict + ] + + for logger in all_loggers: + for handler in getattr(logger, "handlers", []): + if hasattr(handler, "stream"): + original_stream = handler.stream + self.original_handlers_streams.append((handler, original_stream)) + + if original_stream is sys.stdout: + handler.stream = self.captured_stdout + elif original_stream is sys.stderr: + handler.stream = self.captured_stderr + + def _restore_logging_handlers(self): + """Restore original streams to logging handlers.""" + for handler, original_stream in self.original_handlers_streams: + handler.stream = original_stream + self.original_handlers_streams.clear() From 8aa9b770d364686e1798548c54549e630c2fe4e0 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 28 Jul 2026 14:18:53 -0400 Subject: [PATCH 0211/1072] fix: address PR review comments (coderabbit + colleague) - Use pd.to_numeric for TP filtering in analysis.py to handle non-numeric values - Guard against empty ARTIFACT_BASE_DIR in ci.py pipeline failure detection - Anchor resource cleanup suffix match with endswith in prepare_rhaiis.py - Clear _warnings at start of _run_test to prevent cross-invocation leaks - Add pipefail and post-extraction verification to profiler trace copy - Pass restrict_profiles=[workload_key] in per-workload standalone analysis - Deduplicate notification functions into shared _send_alert helper - Stop swallowing exceptions: surface CSV sync, profiler upload failures - Use descriptive artifact dir name with model_key and workload_keys - Factor shared warmup/profiler step args into step_kwargs dict Co-authored-by: Cursor --- projects/rhaiis/orchestration/analysis.py | 2 +- projects/rhaiis/orchestration/ci.py | 5 +- .../rhaiis/orchestration/notifications.py | 91 ++++++++----------- .../rhaiis/orchestration/prepare_rhaiis.py | 4 +- projects/rhaiis/orchestration/test_phase.py | 53 +++++------ .../toolbox/copy_profiler_traces/main.py | 10 +- 6 files changed, 79 insertions(+), 86 deletions(-) diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index f41390aab..df6ee38b2 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -73,7 +73,7 @@ def run_standalone_analysis( (df["version"] == version) & (df["model"] == model_id) & (df["accelerator"] == accelerator) - & (df["TP"].fillna(-1).astype(float).astype(int).astype(str) == tp) + & (pd.to_numeric(df["TP"], errors="coerce").fillna(-1).astype(int).astype(str) == tp) ] if current_rows.empty: diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 9ba9c2e26..c0d70d18b 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -25,7 +25,10 @@ def _check_pipeline_failure_and_notify() -> None: produced FAILURE artifacts or the test step was skipped entirely. """ try: - base_dir = Path(os.environ.get("ARTIFACT_BASE_DIR", "")) + base_dir_env = os.environ.get("ARTIFACT_BASE_DIR", "") + if not base_dir_env: + return + base_dir = Path(base_dir_env) if not base_dir.is_dir(): return diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index 98b35a931..b500188b3 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -11,6 +11,41 @@ logger = logging.getLogger(__name__) +def _send_alert( + error_message: str, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Shared implementation for pipeline failure/warning Slack alerts.""" + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + vllm_args = runtime_config.merge_vllm_args( + runtime_config.get_vllm_defaults(), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + ) + + send_failure_notification( + error=error_message, + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(vllm_args.get("tensor-parallel-size", "")), + dp=str(vllm_args.get("data-parallel-size", "")), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + + def send_pipeline_failure_alert( exc: Exception, *, @@ -19,32 +54,8 @@ def send_pipeline_failure_alert( ) -> None: """Best-effort Slack alert when the pipeline fails.""" try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - - send_failure_notification( - error=traceback.format_exception_only(type(exc), exc)[-1].strip(), - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) + error_message = traceback.format_exception_only(type(exc), exc)[-1].strip() + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) except Exception: logger.warning("Failed to send pipeline failure alert", exc_info=True) @@ -57,32 +68,8 @@ def send_pipeline_warning( ) -> None: """Best-effort Slack alert for non-critical pipeline warnings.""" try: - from projects.core.library import config as _cfg - from projects.rhaiis.postprocess.regression import send_failure_notification - - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator - cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), - model_cfg, - runtime_config.get_workload(workload_keys[0]), - ) - warning_text = "\n".join(f"• {w}" for w in warnings) - send_failure_notification( - error=f"Pipeline completed with warnings:\n{warning_text}", - model=model_cfg.get("hf_model_id", model_key), - accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), - version=_cfg.project.get_config("tests.rhaiis.version", ""), - workload_keys=workload_keys, - cluster=cluster_tag, - ) + error_message = f"Pipeline completed with warnings:\n{warning_text}" + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) except Exception: logger.warning("Failed to send pipeline warning alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index 29d111ff7..ca182f4f3 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -46,13 +46,13 @@ def prepare(): def _delete_resources_by_suffix(resource_type: str, ns: str, suffix: str) -> None: - """Delete resources whose names contain the FJOB suffix.""" + """Delete resources whose names end with the FJOB suffix.""" result = oc("get", resource_type, "-n", ns, "-o", "name", check=False, log_stdout=False) if result.returncode != 0 or not result.stdout: return for line in result.stdout.strip().splitlines(): name = line.strip() - if suffix in name: + if name.endswith(f"-{suffix}"): oc("delete", name, "-n", ns, "--ignore-not-found", check=False) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index f94812476..bef05895d 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -40,7 +40,8 @@ def run( try: _sync_postprocessed_dashboard_csv(model_key, workload_keys) except Exception: - logger.warning("Dashboard CSV S3 sync after postprocessing failed", exc_info=True) + logger.exception("Dashboard CSV S3 sync after postprocessing failed") + ret = 1 return ret @@ -53,7 +54,8 @@ def do_test( deployment_name: str | None = None, ) -> int: try: - with env.NextArtifactDir("testing"): + dir_name = f"test_{model_key}_{'_'.join(workload_keys)}" + with env.NextArtifactDir(dir_name): return _run_test( model_key=model_key, workload_keys=workload_keys, @@ -74,6 +76,7 @@ def _run_test( namespace: str, deployment_name: str | None = None, ) -> int: + _warnings.clear() model_cfg = runtime_config.get_model(model_key) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator @@ -140,7 +143,7 @@ def _run_test( ) logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) except Exception: - logger.debug("Failed to annotate FournosJob with UUID", exc_info=True) + logger.warning("Failed to annotate FournosJob with UUID", exc_info=True) from projects.core.library import config @@ -217,37 +220,29 @@ def _run_test( # Phase 1: warmup or profiler for ALL workloads first for wl_key in workload_keys: - workload = runtime_config.get_workload(wl_key) + step_kwargs = dict( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=runtime_config.get_workload(wl_key), + workload_key=wl_key, + benchmark_timeout=benchmark_timeout, + ) if profiler_enabled: logger.info("Running profiler for workload=%s", wl_key) - _run_profiler_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=wl_key, - benchmark_timeout=benchmark_timeout, - ) + _run_profiler_step(**step_kwargs) elif warmup_enabled: logger.info("Running warmup for workload=%s", wl_key) - _run_warmup_step( - deployment_name=deployment_name, - namespace=namespace, - endpoint_url=endpoint_url, - benchmark_cfg=benchmark_cfg, - model_cfg=model_cfg, - workload=workload, - workload_key=wl_key, - benchmark_timeout=benchmark_timeout, - ) + _run_warmup_step(**step_kwargs) if profiler_enabled: try: _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) except Exception: - logger.warning("Profiler trace upload failed; continuing", exc_info=True) + logger.exception("Profiler trace upload failed") + _warnings.append("Profiler trace upload failed") # Phase 2: benchmark + post-processing for ALL workloads for wl_key in workload_keys: @@ -372,7 +367,13 @@ def _run_workload_benchmark( try: from projects.rhaiis.orchestration.analysis import run_standalone_analysis - run_standalone_analysis(model_cfg, accelerator_key, vllm_args, run_uuid=run_uuid) + run_standalone_analysis( + model_cfg, + accelerator_key, + vllm_args, + run_uuid=run_uuid, + restrict_profiles=[workload_key], + ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) _warnings.append(f"Standalone analysis failed for {workload_key}") diff --git a/projects/rhaiis/toolbox/copy_profiler_traces/main.py b/projects/rhaiis/toolbox/copy_profiler_traces/main.py index 0d9331aad..147fe8dfe 100644 --- a/projects/rhaiis/toolbox/copy_profiler_traces/main.py +++ b/projects/rhaiis/toolbox/copy_profiler_traces/main.py @@ -56,11 +56,13 @@ def list_trace_files(args, context): def copy_traces(args, context): traces_dir = args.artifact_dir / "artifacts/traces" shell.run( - f"oc exec {context.pod_name} -n {args.namespace} " - "-- sh -c 'cd /tmp && tar cf - trace_*.json*' " - f"| tar --no-same-owner -xf - -C {traces_dir}", + 'bash -o pipefail -c "' + f"oc exec {context.pod_name} -n {args.namespace}" + " -- sh -c 'cd /tmp && tar cf - trace_*.json*'" + f' | tar --no-same-owner -xf - -C {traces_dir}"', ) - return f"Copied {context.trace_count} trace files to {traces_dir}" + copied = list(traces_dir.glob("trace_*.json*")) + return f"Copied {len(copied)} trace files to {traces_dir}" if __name__ == "__main__": From 44902bfc63cd0e21fd7dd4ecde617e2fc5b0e4e0 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:12:17 +0300 Subject: [PATCH 0212/1072] Address PR review: stricter error handling in mlflow verifier, fix duplicate directive - mlflow_verifier: raise FileNotFoundError/ValueError instead of returning empty strings on missing vault secrets, missing experiments, no runs, and version extraction failures - mlflow_verifier: suppress InsecureRequestWarning from urllib3 - mlflow_verifier: restore MLFLOW_WORKSPACE env var in finally block - base_verifier: update docstring to reflect exception-based contract - pr_args: remove duplicate /pipeline directive from parsed_directives to avoid double entry in GitHub notifications Co-authored-by: Cursor --- .../caliper/orchestration/mlflow_verifier.py | 67 ++++++++++++------- projects/core/nightly/base_verifier.py | 3 +- projects/mcp_gateway/orchestration/pr_args.py | 3 +- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 8d7d00adb..1941c5490 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -18,6 +18,7 @@ import os import re import time +import warnings from projects.caliper.engine.file_export.mlflow_secrets import ( load_mlflow_secrets_yaml, @@ -62,8 +63,6 @@ def __init__(self, _cfg): def get_last_tested_version(self) -> str: connection = self._load_connection() - if not connection: - return "" last_error = None for attempt in range(1, MAX_ATTEMPTS + 1): @@ -87,47 +86,65 @@ def _load_connection(self) -> dict: secret_path = vault.get_vault_content_path(self.secret_name, self.secret_file) if secret_path is None or not secret_path.exists(): - logger.warning("MLflow secret not found in vault, assuming no previous run") - return {} + raise FileNotFoundError( + f"MLflow secret '{self.secret_name}/{self.secret_file}' not found in vault. " + "Ensure the vault configuration is correct." + ) data = load_mlflow_secrets_yaml(secret_path) validate_mlflow_secrets(data) return data def _query_latest_run(self) -> str: + import urllib3 + import mlflow - if self.workspace: - os.environ["MLFLOW_WORKSPACE"] = self.workspace + warnings.filterwarnings("ignore", category=urllib3.exceptions.InsecureRequestWarning) + + previous_workspace = os.environ.get("MLFLOW_WORKSPACE") + try: + if self.workspace: + os.environ["MLFLOW_WORKSPACE"] = self.workspace - client = mlflow.tracking.MlflowClient() + client = mlflow.tracking.MlflowClient() - experiment = client.get_experiment_by_name(self.experiment) - if experiment is None: - logger.warning("Experiment '%s' not found in MLflow", self.experiment) - return "" + experiment = client.get_experiment_by_name(self.experiment) + if experiment is None: + raise ValueError( + f"MLflow experiment '{self.experiment}' not found. " + "Create the experiment first or check the configuration." + ) - runs = client.search_runs( - experiment_ids=[experiment.experiment_id], - order_by=["start_time DESC"], - max_results=1, - ) + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + order_by=["start_time DESC"], + max_results=1, + ) - if not runs: - logger.info("No runs found in experiment '%s'", self.experiment) - return "" + if not runs: + raise ValueError( + f"No runs found in MLflow experiment '{self.experiment}'." + ) - latest_run = runs[0] - run_name = latest_run.info.run_name or "" - logger.info("Latest MLflow run: %s", run_name) + latest_run = runs[0] + run_name = latest_run.info.run_name or "" + logger.info("Latest MLflow run: %s", run_name) - return self._extract_version(run_name) + return self._extract_version(run_name) + finally: + if previous_workspace is None: + os.environ.pop("MLFLOW_WORKSPACE", None) + else: + os.environ["MLFLOW_WORKSPACE"] = previous_workspace def _extract_version(self, run_name: str) -> str: match = self._version_re.search(run_name) if not match: - logger.warning("Could not extract version from run name: %s", run_name) - return "" + raise ValueError( + f"Could not extract version from run name '{run_name}' " + f"using pattern '{self._version_re.pattern}'." + ) version = match.group(1) diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py index a0c644d6d..de42c2037 100644 --- a/projects/core/nightly/base_verifier.py +++ b/projects/core/nightly/base_verifier.py @@ -16,6 +16,7 @@ class NightlyVerifier(ABC): def get_last_tested_version(self) -> str: """Return the version string of the last successful run. - Returns empty string if no previous run exists. + Raises an exception if the version cannot be determined + (missing credentials, experiment not found, etc.). """ ... diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 1b11aade6..1a6f7d879 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -80,13 +80,14 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st logger.info("Parsed mcp_gateway version: %s", parts[0]) # Parse /pipeline nightly — extract source from pipeline directive + # The directive line itself is already captured by the FOURNOS /pipeline handler, + # so only extract the config overrides here to avoid duplicating it in notifications. if line.startswith("/pipeline nightly "): parts = line.split() if len(parts) == 3: source = parts[2] config_overrides["nightly.source"] = source config_overrides["fournos.job.pipeline_name"] = "nightly" - parsed_directives.append(line) logger.info("Parsed nightly source from pipeline directive: %s", source) return config_overrides, parsed_directives From c4bc7af57f04c58e704f19636debfdfcbaa881d1 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:14:30 +0300 Subject: [PATCH 0213/1072] fix: sort imports in mlflow_verifier to satisfy ruff I001 Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 1941c5490..2ada0b728 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -96,9 +96,8 @@ def _load_connection(self) -> dict: return data def _query_latest_run(self) -> str: - import urllib3 - import mlflow + import urllib3 warnings.filterwarnings("ignore", category=urllib3.exceptions.InsecureRequestWarning) From 024bd90e9c417b73b366d061646f3cda68e85594 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 15:16:31 +0300 Subject: [PATCH 0214/1072] style: apply ruff formatting to mlflow_verifier Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 2ada0b728..7433c9e0b 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -122,9 +122,7 @@ def _query_latest_run(self) -> str: ) if not runs: - raise ValueError( - f"No runs found in MLflow experiment '{self.experiment}'." - ) + raise ValueError(f"No runs found in MLflow experiment '{self.experiment}'.") latest_run = runs[0] run_name = latest_run.info.run_name or "" From 08bf719fb8ddce096193108ba319140ac585b095 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:05:04 +0200 Subject: [PATCH 0215/1072] [caliper] orchestration: postprocess: re-enable the hierarchical KPI format --- projects/caliper/orchestration/postprocess.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index c0a594075..898a9929c 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -527,6 +527,45 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): + # Transform JSONL (schema v1) to hierarchical JSON (schema v2) + try: + logger.info(f"Transforming KPI output to hierarchical format: {output_file}") + + # Read the generated JSONL file + kpis = [] + with open(output_file) as f: + for line in f: + line = line.strip() + if line: + import json + + kpis.append(json.loads(line)) + + if kpis: + # Transform to hierarchical format + hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) + + # Write back as JSON (schema v2) + import json + + with open(output_file, "w") as f: + json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) + + logger.info( + f"Successfully transformed {len(kpis)} KPI records to hierarchical format" + ) + else: + logger.warning("No KPI records found in output file") + + except Exception as transform_error: + logger.error(f"Failed to transform KPIs to hierarchical format: {transform_error}") + return { + "status": "failed", + "error": f"KPI transformation failed: {transform_error}", + "completed_at": time.time(), + "log_file": log_file, + } + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) logger.info( f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" From 81722367aa1027122f2258bbfaaa07a3d6605c26 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 29 Jul 2026 17:38:05 +0300 Subject: [PATCH 0216/1072] add more instructions in the agent.md file and a reviewer skill, you can use lola to adopt it to your coding agnet --- AGENTS.md | 160 ++++++++++++++++++ module/README.md | 138 +++++++++++++++ module/skills/forge-review/SKILL.md | 128 ++++++++++++++ .../forge-review/scripts/gather-diff.sh | 63 +++++++ 4 files changed, 489 insertions(+) create mode 100644 module/README.md create mode 100644 module/skills/forge-review/SKILL.md create mode 100755 module/skills/forge-review/scripts/gather-diff.sh diff --git a/AGENTS.md b/AGENTS.md index b2043d04f..1bc912ca4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,3 +87,163 @@ logger.info(f"Applying secret: {manifest}") # LEAKS if manifest has sensitive d # BAD: Including tokens in error messages raise RuntimeError(f"Failed to auth with token {token}") # LEAKS TOKEN ``` + +## Error Handling: Never Swallow Exceptions + +**Never catch-and-warn silently.** A `logger.warning(...)` inside a bare `except` gets buried in logs and failures go unnoticed for weeks. + +### Rules + +1. **Raise typed exceptions.** Use `ValueError`, `FileNotFoundError`, `RuntimeError` — not `logger.warning()` + `return ""`. + A missing config, missing vault secret, or failed version extraction is a real error. + +2. **Register visible notifications for operational failures.** + Use `ci.add_notification_file(name, message)` so failures appear in GitHub notifications, + not just in log streams nobody reads. + +3. **Bootstrap edge cases belong in the project, not shared libraries.** + If a new nightly test has no previous MLflow run yet, the *project orchestration* should + handle that gracefully (e.g., catch the exception and treat it as "no previous version"). + Shared/library code should raise — callers decide policy. + +### Forbidden + +```python +# BAD: Exception swallowed, failure invisible +try: + upload_traces(data) +except Exception: + logger.warning("Upload failed; continuing", exc_info=True) + +# BAD: Returning empty string instead of raising +if connection is None: + return "" # caller has no idea something broke +``` + +### Correct + +```python +# GOOD: Raise so caller can decide +if connection is None: + raise ValueError("MLflow connection failed — check vault config") + +# GOOD: If truly optional, still make it visible +try: + upload_traces(data) +except Exception: + logger.exception("Trace upload failed") + ci.add_notification_file("trace-upload-failed", "Profiler trace upload failed — check logs") + raise +``` + +## Layer Isolation: Toolbox Independence + +Toolbox scripts are the **lower layer** — standalone, reusable cluster-state commands. +Orchestration is the **upper layer** — CI phases, config, presets. + +### Rules + +1. **Toolbox must never import from orchestration.** + No `from projects.NAME.orchestration.runtime_config import cfg` in toolbox code. + All inputs come through the `run()` entrypoint parameters for strong isolation. + +2. **Pass all values explicitly via `run()` parameters.** + Declare typed parameters with defaults in the entrypoint. This makes the command + standalone, testable, and reviewable via `_meta/metadata.yaml`. + +3. **Manifests and templates live with their toolbox command.** + Put them in `projects/NAME/toolbox/COMMAND_NAME/templates/` — keeps the command self-contained. + +### Forbidden + +```python +# BAD: Toolbox importing orchestration config +from projects.llamastack.orchestration.runtime_config import cfg +hpa_cfg = cfg.get_hpa_config() +min_replicas = hpa_cfg.get("min_replicas", 1) +``` + +### Correct + +```python +# GOOD: All config passed through entrypoint +@entrypoint +def run(*, min_replicas: int = 1, max_replicas: int = 4, memory_target: int = 75): + """ + Configure HPA for the deployment. + + Args: + min_replicas: Minimum replica count + max_replicas: Maximum replica count + memory_target: Target memory utilization percentage + """ + return execute_tasks(locals()) +``` + +## Waiting for State: No `time.sleep()` + +### Rules + +1. **Never use `time.sleep(N)` to wait for cluster state changes.** + Arbitrary sleeps are unreliable — too short in slow environments, wasted time in fast ones. + +2. **Use `@retry` to poll for readiness.** + +```python +# GOOD: Poll until resource is ready +@retry(attempts=60, delay=30, backoff=1.0) +@task +def wait_until_ready(args, ctx): + status = oc_get_json("datasciencecluster", "default-dsc")["status"]["phase"] + return status == "Ready" +``` + +3. **Reuse existing wait toolbox scripts.** + Before writing a new wait loop, check `projects/rhoai/toolbox/`, `projects/rhaiis/toolbox/`, + `projects/core/` — many wait patterns already exist. + +## Config Types: No Unnecessary Coercion + +`config.project.get_config()` returns typed values from YAML. Do not cast. + +```python +# BAD: YAML already returns int +replicas = int(config.project.get_config("runtime.replicas")) + +# BAD: Custom bool coercion for YAML values +def _as_bool(value): + return value.strip().lower() not in ("false", "0", "no") + +# GOOD: Trust YAML types +replicas = config.project.get_config("runtime.replicas") +``` + +If a config value has the wrong type, fix the YAML — not the code. + +## Don't Duplicate Framework Behavior + +Before implementing preset handling, vault init, phase registration, or PR arg parsing — +check what `config.init()`, `ci_base`, and core library already provide. + +- Preset application: handled by `config.init()` +- Config overrides: handled by `config.project.apply_config_overrides()` +- `/test` directives: handled at framework level + +Duplicating framework behavior leads to drift and subtle bugs. + +## Environment Variable Hygiene + +If you temporarily set an environment variable, restore it in a `finally` block. + +```python +# GOOD: Cleanup in finally +old_value = os.environ.get("MLFLOW_WORKSPACE") +try: + os.environ["MLFLOW_WORKSPACE"] = workspace + # ... do work ... +finally: + if old_value is None: + os.environ.pop("MLFLOW_WORKSPACE", None) + else: + os.environ["MLFLOW_WORKSPACE"] = old_value +``` diff --git a/module/README.md b/module/README.md new file mode 100644 index 000000000..d371b8467 --- /dev/null +++ b/module/README.md @@ -0,0 +1,138 @@ +# FORGE AI Context Module + +This module contains reusable AI agent skills for working with the FORGE codebase. +Skills follow the [AgentSkills.io](https://agentskills.io/specification) standard and +can be installed into any supported coding agent using [Lola](https://github.com/LobsterTrap/lola). + +## Available Skills + +| Skill | Description | +|-------|-------------| +| `forge-review` | Pre-commit review that checks local changes against FORGE rules in `AGENTS.md` | + +## Installation + +### Prerequisites + +Install Lola: + +```bash +# Using uv (recommended) +uv tool install lola-ai + +# Or using pip +pip install lola-ai +``` + +### Register the module + +From the forge repository root: + +```bash +lola mod add . +``` + +### Install into your project + +**Important:** Run these commands from the forge project directory so skills are +installed at the project level, not globally. + +#### Cursor + +```bash +cd /path/to/forge +lola install forge -a cursor +``` + +This creates: +``` +forge/ + .cursor/ + skills/ + forge-review/ + SKILL.md + scripts/ + gather-diff.sh +``` + +The skill will appear in Cursor's skill list and can be triggered by asking +the agent to "review my changes" or "forge review" before committing. + +#### Claude Code + +```bash +cd /path/to/forge +lola install forge -a claude-code +``` + +This creates: +``` +forge/ + .claude/ + skills/ + forge-review/ + SKILL.md + scripts/ + gather-diff.sh +``` + +The skill is available in Claude Code when working inside the forge project. + +#### All detected agents at once + +```bash +cd /path/to/forge +lola install forge +``` + +Lola auto-detects which agents are available and installs to all of them. + +### Declarative installation (team workflow) + +Add a `.lola-req` file to the forge project root: + +``` +# .lola-req — FORGE AI context module +. +``` + +Then any team member clones the repo and runs: + +```bash +lola sync +``` + +This installs the module into whichever agents they have configured. + +## Manual installation (without Lola) + +If you prefer not to use Lola, copy the skill directory manually. + +#### Cursor + +```bash +mkdir -p .cursor/skills/forge-review +cp -r module/skills/forge-review/* .cursor/skills/forge-review/ +``` + +#### Claude Code + +```bash +mkdir -p .claude/skills/forge-review +cp -r module/skills/forge-review/* .claude/skills/forge-review/ +``` + +## Updating + +After pulling new changes that update the module: + +```bash +lola mod update forge +lola install forge +``` + +Or with declarative mode: + +```bash +lola sync +``` diff --git a/module/skills/forge-review/SKILL.md b/module/skills/forge-review/SKILL.md new file mode 100644 index 000000000..e09153cfa --- /dev/null +++ b/module/skills/forge-review/SKILL.md @@ -0,0 +1,128 @@ +--- +name: forge-review +description: > + Pre-commit review that diffs local changes against the remote branch and checks + for FORGE AGENTS.md rule violations. Use before committing, when the user says + "review my changes", "check before commit", "pre-commit review", or "forge review". +--- + +# FORGE Pre-Commit Review + +Review local code changes against the FORGE project rules defined in `AGENTS.md` before committing. + +## Trigger + +Run this review **before every commit**. If the user asks to commit, run this review first and present findings before proceeding. + +## Procedure + +### Step 1: Gather the diff + +Run the diff gathering script: + +```bash +./scripts/gather-diff.sh +``` + +The script outputs structured sections: +- `=== DIFF_META ===` — branch name, remote, timestamp +- `=== STATUS ===` — `git status --short` +- `=== DIFF_TYPE: staged|unstaged|branch|none ===` — the actual diff, prioritizing staged > unstaged > branch +- `=== UNSTAGED_WARNING ===` — warns if unstaged changes exist alongside staged (won't be committed) +- `=== UNTRACKED ===` — new files not yet tracked by git + +For any files listed under `UNTRACKED`, read their full content — they won't appear in the diff output. + +### Step 2: Load the rules + +Read the `AGENTS.md` file from the repository root. This contains all project-wide rules that code must follow. Parse each `##` section as a distinct rule category. + +### Step 3: Analyze each rule against the diff + +For every added or modified line in the diff (`+` lines), check against each rule category: + +#### Secret Handling (CRITICAL — blocks commit) +- `oc_apply()` called with a path under `ARTIFACT_DIR` on a Secret-kind manifest +- Secret data logged via `logger.info/debug/warning(f"...{secret}...")` +- Tokens, passwords, credentials in error messages or `raise` statements +- Environment variables used to pass secrets between components +- Pod env vars with inline secret values instead of Secret references +- Any manifest with `kind: Secret` written to an artifact path + +#### Error Handling +- Bare `except Exception` that only logs a warning and continues +- Functions returning empty string `""` or `None` on failure instead of raising — **except** inside + `@retry`-decorated tasks, where returning a falsy value (`False`, `None`, `""`, `[]`) is the + intended mechanism to trigger a retry. Only flag falsy returns as error swallowing when the + function does NOT have a `@retry` decorator above `@task`. +- `logger.warning(...)` as the sole response to a failure condition +- Missing `ci.add_notification_file()` for operational failures that need visibility + +#### Layer Isolation +- Any file under `projects/*/toolbox/` importing from `projects/*/orchestration/` +- Toolbox code reading config via `config.project.get_config()` directly instead of receiving values through `run()` parameters +- Manifest/template files placed outside their toolbox command directory + +#### No time.sleep() +- `time.sleep(N)` used to wait for cluster state (pod ready, operator deployed, resource status) +- Any hardcoded delay where `@retry` polling should be used instead +- When flagging `time.sleep()`, check whether the containing task already has `@retry`. If not, + suggest adding `@retry(attempts=N, delay=M, backoff=1.0)` above `@task` and replacing the + sleep with a polling check that returns falsy to trigger retry + +#### Config Type Coercion +- `int()`, `float()`, `bool()`, `str()` wrapping `config.project.get_config()` return values +- Custom bool/int coercion helpers for YAML config values + +#### Framework Duplication +- Re-implementing preset handling that `config.init()` already does +- Manual vault initialization that duplicates `vault.init()` +- Custom `/test` or `/pipeline` directive parsing that framework handles + +#### Environment Variable Hygiene +- `os.environ[KEY] = value` without a corresponding `finally` cleanup +- Temporary env var mutation without saving and restoring the original value + +### Step 4: Report findings + +Format each finding as: + +``` +:: : . . +``` + +Severity levels: +- `🔴 BLOCK` — Secret handling violations. **Must fix before commit.** +- `🟡 WARN` — Error swallowing, layer violations, sleep usage. Should fix. +- `🔵 NOTE` — Type coercion, env hygiene, framework duplication. Improve if easy. + +#### Output structure + +``` +## FORGE Pre-Commit Review + +### Diff summary + + +### Findings + + +### Verdict +✅ PASS — no rule violations found, safe to commit. +⚠️ WARN — warnings found. Review recommended before commit. +🚫 BLOCK — critical violations. Must fix before commit. +``` + +If verdict is PASS, proceed with the commit. If BLOCK, do not commit — show findings and offer to fix. If WARN, show findings and ask the user whether to proceed or fix first. + +### Step 5: Offer fixes + +For each finding, offer a concrete fix. If the user agrees, apply fixes and re-run the review to confirm the violations are resolved before committing. + +## Scope + +Only review files under `projects/`. Ignore: +- `docs/` (documentation only) +- `fournos/gitops/` (Tekton pipelines, different patterns) +- Test files (`test_*.py`, `conftest.py`) — unless they contain secret handling code +- Configuration files (`*.yaml`, `*.json`) — unless they contain inline secrets diff --git a/module/skills/forge-review/scripts/gather-diff.sh b/module/skills/forge-review/scripts/gather-diff.sh new file mode 100755 index 000000000..c93cf6488 --- /dev/null +++ b/module/skills/forge-review/scripts/gather-diff.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Gather the diff that represents what the user intends to commit. +# Priority: staged > unstaged > branch diff against remote. +# Outputs structured sections so the reviewing agent can parse reliably. + +BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "HEAD") +REMOTE_BRANCH="origin/${BRANCH}" + +echo "=== DIFF_META ===" +echo "branch: ${BRANCH}" +echo "remote: ${REMOTE_BRANCH}" +echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + +echo "" +echo "=== STATUS ===" +git status --short + +STAGED=$(git diff --cached) +UNSTAGED=$(git diff) + +if [[ -n "${STAGED}" ]]; then + echo "" + echo "=== DIFF_TYPE: staged ===" + echo "${STAGED}" + + if [[ -n "${UNSTAGED}" ]]; then + echo "" + echo "=== UNSTAGED_WARNING ===" + echo "There are also unstaged changes that will NOT be part of this commit:" + git diff --stat + fi +elif [[ -n "${UNSTAGED}" ]]; then + echo "" + echo "=== DIFF_TYPE: unstaged ===" + echo "${UNSTAGED}" +else + HAS_REMOTE=$(git ls-remote --heads origin "${BRANCH}" 2>/dev/null | head -1) + if [[ -n "${HAS_REMOTE}" ]]; then + BRANCH_DIFF=$(git diff "${REMOTE_BRANCH}...HEAD" 2>/dev/null || true) + if [[ -n "${BRANCH_DIFF}" ]]; then + echo "" + echo "=== DIFF_TYPE: branch ===" + echo "${BRANCH_DIFF}" + else + echo "" + echo "=== DIFF_TYPE: none ===" + echo "No changes found between local and remote." + fi + else + echo "" + echo "=== DIFF_TYPE: none ===" + echo "No remote branch and no local changes." + fi +fi + +UNTRACKED=$(git ls-files --others --exclude-standard) +if [[ -n "${UNTRACKED}" ]]; then + echo "" + echo "=== UNTRACKED ===" + echo "${UNTRACKED}" +fi From b61e356becaaa72727dda419f768ee17e1f0e0be Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 13:39:08 -0400 Subject: [PATCH 0217/1072] feat: add SGLang and TRT-LLM engine support Replace all vllm-specific references with generic engine abstraction. Configuration, runtime helpers, CLI, templates, and callers now support vllm, sglang, and trtllm engines with automatic argument translation. Fix parity gaps with model-furnace: add securityContext, XDG env vars, NCCL_DEBUG for multi-GPU, 64Gi shared memory, TRT-LLM termination grace period and resource limits, and disable prometheus for SGLang. Co-authored-by: Cursor --- projects/rhaiis/README.md | 251 ++++++++++++------ projects/rhaiis/orchestration/analysis.py | 24 +- projects/rhaiis/orchestration/ci.py | 4 +- projects/rhaiis/orchestration/cli.py | 45 +++- .../rhaiis/orchestration/config.d/models.yaml | 56 ++-- .../rhaiis/orchestration/config.d/rhaiis.yaml | 34 ++- .../rhaiis/orchestration/notifications.py | 17 +- .../orchestration/presets.d/presets.yaml | 10 + .../rhaiis/orchestration/runtime_config.py | 83 +++++- projects/rhaiis/orchestration/test_phase.py | 102 ++++--- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 13 +- .../templates/inferenceservice.yaml.j2 | 29 +- .../templates/servingruntime.yaml.j2 | 129 ++++++++- 13 files changed, 595 insertions(+), 202 deletions(-) diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index f0e425e8f..1ccc40af6 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -4,30 +4,46 @@ using KServe InferenceService. The current implementation focuses on single-replica vLLM deployments benchmarked -with GuideLLM. The workflow deploys an InferenceService, waits for readiness, -runs a GuideLLM benchmark job, extracts results, and cleans up. +with GuideLLM. The workflow deploys an InferenceService, runs warmup and/or +profiler passes, executes multi-profile benchmarks, generates a dashboard CSV via +the Caliper postprocessing pipeline, syncs results to S3, and cleans up. ## Workflow sequence ``` -deploy_kserve_isvc Deploy ServingRuntime + InferenceService +deploy_kserve_isvc Deploy ServingRuntime + InferenceService | -wait_isvc_ready Poll InferenceService status + health check +wait_isvc_ready Poll InferenceService status + health check | -run_guidellm_benchmark Create PVC, run GuideLLM job, copy results, cleanup + ┌─────┴─────┐ + │ Phase 1 │ For each workload profile: + │ (warmup │ - warmup run (optional) OR profiler-gated run + │ /profiler)│ - profiler trace upload to S3 + └─────┬─────┘ + ┌─────┴─────┐ + │ Phase 2 │ For each workload profile (in its own artifact dir): + │ (benchmark)│ - write per-workload test labels + │ │ - run GuideLLM benchmark at configured rates + └─────┬─────┘ | -capture_isvc_state Capture ISVC YAML, pod logs, events +capture_isvc_state Capture ISVC YAML, pod logs, events | -cleanup_isvc Delete InferenceService + ServingRuntime +cleanup_isvc Delete InferenceService + ServingRuntime + | +Caliper postprocessing Parse → KPI → CSV export (per-profile rows) + | +S3 dashboard sync Append CSV to consolidated dashboard on S3 + | +Regression analysis Compare against baseline version (optional) ``` On failure at any step, `capture_isvc_state` and `cleanup_isvc` still run -(try/finally in the orchestration layer). +(try/finally in the orchestration layer). Pipeline failures and non-critical +warnings are reported via Slack. -After the benchmark, `run_and_postprocess()` automatically runs: -- KPI computation (22 metrics via rhaiis postprocess plugin) -- PSAP payload generation (model-furnace compatible JSON) -- MLflow metadata tagging +Each workload profile runs in its own `NextArtifactDir` with per-workload test +labels, so the Caliper parser creates separate records per profile. This ensures +each profile gets its own dashboard CSV rows. The benchmark step uses the canonical `projects.guidellm.toolbox.run_guidellm_benchmark` (shared with llm_d). rhaiis builds `guidellm_args` from its config and passes them @@ -37,8 +53,8 @@ to the canonical runner. Config is split into `config.yaml` (base) and `config.d/` (per-domain): -- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper -- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types +- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper postprocessing +- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types, S3 settings, profiler - [`orchestration/config.d/models.yaml`](./orchestration/config.d/models.yaml) — 52 model definitions - [`orchestration/config.d/workloads.yaml`](./orchestration/config.d/workloads.yaml) — workload profiles @@ -46,11 +62,14 @@ Key sections: | Section | Purpose | |---------|---------| -| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args | +| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args, S3 config | +| `rhaiis.s3` | S3 bucket, vault, and credentials for dashboard CSV and profiler trace uploads | +| `rhaiis.profiler` | PyTorch profiler settings (enable, S3 prefix, rates, labels) | | `models` | Model definitions (hf_model_id, per-model vLLM overrides) | | `workloads` | Benchmark profiles (data shape, rates, max_seconds) | -| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size | -| `tests` | CI test mapping (model_key, workload_key) | +| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret | +| `tests` | CI test mapping (model_key, workload_keys, version) | +| `caliper.postprocess` | Caliper postprocessing pipeline (parse, KPI, CSV export) | ## Fournos integration @@ -58,38 +77,60 @@ rhaiis supports Fournos-driven execution via `ci.py`: ``` bin/run_ci rhaiis ci resolve-fournos-config # Populate spec.secretRefs + hardware -bin/run_ci rhaiis ci pre-cleanup # Delete leftover jobs/pods +bin/run_ci rhaiis ci pre-cleanup # No-op (avoid cleaning running resources) bin/run_ci rhaiis ci prepare # Verify cluster, ensure namespace/SA +bin/run_ci rhaiis ci preflight # Preflight checks bin/run_ci rhaiis ci test # Deploy, benchmark, capture, cleanup +bin/run_ci rhaiis ci post-cleanup # Detect pipeline failures, cleanup resources bin/run_ci rhaiis ci export-artifacts # Caliper export to MLflow ``` -FournosJob example: +### FournosJob YAML + ```yaml apiVersion: fournos.dev/v1 kind: FournosJob metadata: generateName: rhaiis-benchmark- spec: - owner: mehul - displayName: rhaiis-qwen3-benchmark - pipeline: forge-prepare-test - cluster: fournos-fg - priority: manual - hardware: - gpuType: l40s - gpuCount: 1 + owner: haumesh + displayName: rhaiis-benchmark + pipeline: forge-full + cluster: zeus executionEngine: forge: project: rhaiis - args: [nvidia, profile1] + args: [nvidia] configOverrides: - tests.rhaiis.model_key: qwen3-0_6b - rhaiis.namespace: kserve-e2e-perf + tests.rhaiis.model_key: nemotron3super-120b-fp8 + rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 + tests.rhaiis.version: "vLLM-0.24.0" + tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] + rhaiis.vllm_args.tensor-parallel-size: 2 + rhaiis.cluster_tag: "zeus2" rhaiis.deploy.image_pull_secret: npalaska-image-pull - rhaiis.deploy.storage_pvc: model-pvc + caliper.postprocess.csv_dashboard.enabled: true + env: + PULL_PULL_SHA: "" ``` +### GitHub PR workflow + +Jobs can also be triggered via PR comments on `openshift-psap/forge`: + +``` +/test fournos rhaiis nvidia +/pipeline forge-full +/cluster zeus +/var tests.rhaiis.model_key: nemotron3super-120b-fp8 +/var tests.rhaiis.version: vLLM-0.24.0 +/var tests.rhaiis.workload_keys: ["profile1"] +/var rhaiis.vllm_args.tensor-parallel-size: 2 +``` + +Note: `/var` directives use `key: value` format (colon required). +If `PULL_PULL_SHA` is not set, the PR HEAD commit is used automatically. + Presets from `args` are applied via `project.args` → `presets.d/presets.yaml`. Config overrides use dot-notation to set any nested config value. @@ -98,13 +139,23 @@ Available configOverrides: | Key | Description | |-----|-------------| | `tests.rhaiis.model_key` | Model key from config.d/models.yaml | +| `tests.rhaiis.version` | Version label for dashboard and regression | +| `tests.rhaiis.workload_keys` | List of workload profiles to run | +| `tests.rhaiis.warmup` | Enable warmup pass before benchmarks | +| `tests.rhaiis.slack_user` | Slack user ID for failure notifications | +| `tests.rhaiis.compare_version` | Baseline version for regression comparison | | `rhaiis.namespace` | Kubernetes namespace | +| `rhaiis.cluster_tag` | Cluster identifier for dashboard grouping | | `rhaiis.deploy.image_pull_secret` | Image pull secret name | | `rhaiis.deploy.storage_pvc` | PVC name for model storage | | `rhaiis.deploy.replicas` | Number of predictor replicas | | `rhaiis.images.nvidia` | vLLM container image override | | `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | +| `rhaiis.profiler.enabled` | Enable PyTorch profiler | +| `rhaiis.agent_analysis.enabled` | Enable AI agent regression analysis | +| `caliper.postprocess.csv_dashboard.enabled` | Enable dashboard CSV S3 sync | | `benchmarks.guidellm.timeout` | Benchmark timeout in seconds | +| `benchmarks.guidellm.hf_token_secret` | K8s secret name for HF_TOKEN (omit to skip) | Monitoring Fournos jobs: ```bash @@ -122,16 +173,22 @@ oc patch fournosjob -n psap-automation \ - CLI: [`orchestration/cli.py`](./orchestration/cli.py) - CI: [`orchestration/ci.py`](./orchestration/ci.py) (Fournos pipeline) - CI test: [`orchestration/test_rhaiis.py`](./orchestration/test_rhaiis.py) +- Test phase: [`orchestration/test_phase.py`](./orchestration/test_phase.py) +- Analysis: [`orchestration/analysis.py`](./orchestration/analysis.py) +- Notifications: [`orchestration/notifications.py`](./orchestration/notifications.py) ## Toolbox commands | Command | Source | Purpose | |---------|--------|---------| -| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime | +| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime with configurable labels | | `wait_isvc_ready` | [rhaiis](./toolbox/wait_isvc_ready/) | Poll InferenceService readiness with health check | | `run_guidellm_benchmark` | [canonical](../guidellm/toolbox/run_guidellm_benchmark/) | Run GuideLLM benchmark (shared with llm_d) | | `capture_isvc_state` | [rhaiis](./toolbox/capture_isvc_state/) | Capture InferenceService YAML, pod logs, events, describe output | | `cleanup_isvc` | [rhaiis](./toolbox/cleanup_isvc/) | Delete InferenceService, ServingRuntime, wait for deletion | +| `enable_profiler_gate` | [rhaiis](./toolbox/enable_profiler_gate/) | Enable/disable the PyTorch profiler gate file on the vLLM pod | +| `verify_profiler_prereqs` | [rhaiis](./toolbox/verify_profiler_prereqs/) | Verify profiler prerequisites (gate file, sitecustomize.py) | +| `copy_profiler_traces` | [rhaiis](./toolbox/copy_profiler_traces/) | Copy Chrome trace JSON files from the vLLM pod | ## Usage @@ -208,68 +265,66 @@ Available overrides: ## Result extraction -GuideLLM results are extracted using the copy-pod pattern (same as llm_d): +GuideLLM results are extracted using `oc cp` with a gzip fallback for large files: 1. GuideLLM job writes `benchmarks.json` to a PVC mounted at `/results` 2. A copy pod is created on the same node (required for ReadWriteOnce PVC) -3. Results are extracted via `oc exec cat /results/benchmarks.json` -4. Written to local `artifacts/results/benchmarks.json` +3. Results are copied via `oc cp` from the copy pod +4. If `oc cp` fails for large files (e.g. `unexpected EOF`), the file is gzipped + inside the pod, copied compressed, and decompressed locally 5. PVC, job, and copy pod are deleted -Artifacts are stored under `/tmp/forge_/002__run_guidellm_benchmark/artifacts/`. +## Postprocessing and dashboard CSV -## Available models +After benchmarks complete, `run_and_postprocess()` runs the Caliper pipeline: -52 models from model_furnace are defined in `config.yaml`. Key families: +1. **Parse**: `RhaiisParser` extends `GuideLLMParser` with extra metrics (p1/p999 + percentiles, mean latencies, token counts, throughput) from raw `benchmarks.json` +2. **KPI generate**: `RhaiisKpiHandler` emits 33 KPI types per rate point per profile +3. **CSV export**: `RhaiisPlugin.export_kpis_to_csv` maps KPIs to the dashboard CSV + schema with all metadata columns (model, version, TP, accelerator, UUID, etc.) +4. **S3 sync**: The CSV is appended to the consolidated dashboard CSV on S3 +5. **Regression check**: Optionally compares current vs. baseline version -| Family | Key examples | TP size | -|--------|-------------|---------| -| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | -| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | -| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | -| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | -| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | -| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | -| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | -| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | -| Validation | `qwen3-0_6b` | 1 | +Dashboard CSV columns include throughput (output tok/s, total tok/s), latency +percentiles (TTFT/TPOT/ITL at median, p1, p95, p99, p999, mean), request latency, +token counts, concurrency, and run metadata. -Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` +## Notifications -## Workload profiles +Pipeline events are reported to Slack via the topsail bot: -From model_furnace `guidellm_profiles.iterations`: +- **Failure alerts**: Sent when the pipeline crashes (exceptions, image pull failures, + infrastructure errors). Includes error details, model, accelerator, TP, version, + and job ID. +- **Warning alerts**: Sent when the pipeline completes with non-critical warnings + (e.g. profiler trace upload failed, predictor log upload failed). +- **Regression alerts**: Sent when regression analysis detects performance changes, + with a link to the dashboard filtered by the relevant configuration. -| Key | Prompt tokens | Output tokens | Rates | Max seconds | -|-----|--------------|---------------|-------|-------------| -| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | -| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | -| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | -| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | +Notifications are sent to the `psap-rhaiis-alerts` Slack channel. The post-cleanup +step also detects infrastructure failures that occur before the test step runs. -## Postprocessing and KPIs +## PyTorch profiling -After each benchmark run, the rhaiis postprocess plugin (`projects/rhaiis/postprocess/`) -automatically computes 22 KPIs with model-furnace parity: +When `rhaiis.profiler.enabled: true`, the pipeline runs profiler-gated benchmarks +before the main benchmarks: -| Category | KPIs | -|----------|------| -| Throughput | request_rate, tokens/s, input_tokens/s, output_tokens/s | -| Latency | ttft (median, p95, p99), itl (median, p95, p99), tpot (median, p95, p99) | -| Request latency | request_latency (median, p95) | -| Token efficiency | input_tokens/request, output_tokens/request | -| Request stats | concurrency, completed_requests, failed_requests | -| Benchmark | duration, prompt_token_count_mean | +1. Verify profiler prerequisites on the vLLM pod (gate file, sitecustomize.py) +2. Enable the profiler gate with a workload-specific label (e.g. `isl1000_osl1000`) +3. Run a GuideLLM benchmark at the configured profiler rates +4. Disable the profiler gate +5. Copy Chrome trace JSON files from the vLLM pod +6. Upload traces to S3 organized by accelerator/model/TP/version/profile -KPIs are written to `kpis.json` in the postprocess output directory. +Traces are viewable in `chrome://tracing` or Perfetto UI. -## PSAP payload +## Parallel job isolation -A model-furnace compatible `PSAP_perf____.json` -is generated after each benchmark. It wraps the raw `benchmarks.json` with experiment -metadata (model, vLLM image/args, accelerator, timestamps) for downstream compatibility. +Multiple FournosJobs can run concurrently on the same cluster. Each job gets a +unique deployment name by appending the last 5 characters of the FJOB name +(e.g. `nvidia-nemotron-3-super-120b-a1-54r5f`). Resource cleanup is scoped to +the current job's suffix to avoid interfering with other running jobs. ## MLflow export @@ -281,18 +336,48 @@ MLflow tags set on each run: | Tag | Example | |-----|---------| | `project` | rhaiis | -| `model_key` | qwen3-0_6b | -| `hf_model_id` | Qwen/Qwen3-0.6B | +| `model_key` | nemotron3super-120b-fp8 | +| `hf_model_id` | nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 | | `accelerator` | nvidia | -| `tensor_parallel_size` | 1 | -| `vllm_image` | quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2 | -| `vllm_version` | 3.4.0-ea.2-1773886296 | -| `workload_key` | profile1 | -| `rates` | 1,5,10 | +| `tensor_parallel_size` | 2 | +| `vllm_image` | vllm/vllm-openai:v0.24.0 | +| `vllm_version` | v0.24.0 | +| `workload_key` | profile1,profile2,profile4 | +| `rates` | 1,50,100,200,300 | | `guidellm_backend_type` | openai_http | MLflow experiment: `forge-rhaiis` +## Available models + +52 models are defined in `config.d/models.yaml`. Key families: + +| Family | Key examples | TP size | +|--------|-------------|---------| +| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | +| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | +| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | +| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | +| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | +| Nemotron 120B | `nemotron3super-120b-fp8` | 2 | +| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | +| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | +| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | +| Validation | `qwen3-0_6b` | 1 | + +Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` + +## Workload profiles + +| Key | Prompt tokens | Output tokens | Rates | Max seconds | +|-----|--------------|---------------|-------|-------------| +| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | +| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | +| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | +| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | + ## Presets Presets in `presets.d/presets.yaml` provide shortcuts for common configurations: diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index df6ee38b2..724b99b5f 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -15,7 +15,7 @@ def run_standalone_analysis( model_cfg: dict, accelerator_key: str, - vllm_args: dict, + engine_args: dict, *, run_uuid: str = "", restrict_profiles: list[str] | None = None, @@ -67,7 +67,12 @@ def run_standalone_analysis( df[col] = df[col].str.strip() model_id = model_cfg.get("hf_model_id", "") - tp = str(vllm_args.get("tensor-parallel-size", 1)) + tp = str( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) current_rows = df[ (df["version"] == version) @@ -99,7 +104,7 @@ def run_standalone_analysis( accelerator, run_uuid=run_uuid, restrict_profiles=restrict_profiles, - vllm_args=vllm_args, + engine_args=engine_args, ) except Exception: logger.warning("Standalone analysis failed", exc_info=True) @@ -119,7 +124,7 @@ def run_regression_check( *, run_uuid: str = "", restrict_profiles: list[str] | None = None, - vllm_args: dict | None = None, + engine_args: dict | None = None, ) -> None: from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials from projects.core.library import config @@ -176,7 +181,9 @@ def run_regression_check( from projects.rhaiis.postprocess.regression import send_regression_notification - _vllm = vllm_args or {} + _ea = engine_args or {} + tp = _ea.get("tensor-parallel-size") or _ea.get("tp-size") or _ea.get("tp_size") or "" + dp = _ea.get("data-parallel-size") or _ea.get("dp-size") or "" send_regression_notification( analysis, model=model_cfg.get("hf_model_id", ""), @@ -185,8 +192,8 @@ def run_regression_check( slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), notification_vault="psap-forge-notifications", report_url=report_url, - tp=str(_vllm.get("tensor-parallel-size", "")), - dp=str(_vllm.get("data-parallel-size", "")), + tp=str(tp), + dp=str(dp), ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) @@ -233,7 +240,8 @@ def run_agent_analysis( logger.warning("Agent not reachable, skipping analysis: %s", detail) return "" - tp = str(model_cfg.get("vllm_args", {}).get("tensor-parallel-size", 1)) + ea = model_cfg.get("engine_args", {}) + tp = str(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) model = model_cfg.get("hf_model_id", "") improvements = analysis.get("improvements", []) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index c0d70d18b..89a72f62e 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -97,8 +97,8 @@ def resolve_hardware_request(hardware_spec: dict) -> dict: model_key = runtime_config.get_test_model_key() model = runtime_config.get_model(model_key) - vllm_args = model.get("vllm_args", {}) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) + ea = model.get("engine_args", {}) + tp_size = int(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) diff --git a/projects/rhaiis/orchestration/cli.py b/projects/rhaiis/orchestration/cli.py index b65a07c54..cea6215e6 100644 --- a/projects/rhaiis/orchestration/cli.py +++ b/projects/rhaiis/orchestration/cli.py @@ -51,7 +51,13 @@ def cli(ctx): @click.option("--namespace", "-n", default=None, help="Kubernetes namespace") @click.option("--deployment-name", default=None, help="Deployment name (defaults to model name)") @click.option("--accelerator", type=click.Choice(["nvidia", "amd"]), default=None) -@click.option("--vllm-image", help="vLLM container image override") +@click.option( + "--engine", + type=click.Choice(["vllm", "sglang", "trtllm"]), + default=None, + help="Inference engine", +) +@click.option("--serving-image", help="Serving engine container image override") @click.option("--tensor-parallel", "-tp", type=int, help="Tensor parallel size override") @click.option("--replicas", "-r", type=int, default=None) @click.option("--storage-source", type=click.Choice(["hf", "pvc"]), default=None) @@ -70,7 +76,8 @@ def test( namespace: str | None, deployment_name: str | None, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -93,7 +100,8 @@ def test( _apply_cli_overrides( workload_key=workload_key, accelerator=accelerator, - vllm_image=vllm_image, + engine=engine, + serving_image=serving_image, tensor_parallel=tensor_parallel, replicas=replicas, storage_source=storage_source, @@ -128,7 +136,8 @@ def _apply_cli_overrides( *, workload_key: str, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -140,11 +149,21 @@ def _apply_cli_overrides( ) -> None: if accelerator: config.project.set_config("rhaiis.accelerator", accelerator) - if vllm_image: + if engine: + config.project.set_config("rhaiis.engine", engine) + resolved_engine = runtime_config.get_engine() + if serving_image: resolved_accel = runtime_config.get_accelerator() - config.project.set_config(f"rhaiis.images.{resolved_accel}", vllm_image) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.images.{resolved_accel}", serving_image + ) if tensor_parallel is not None: - config.project.set_config("rhaiis.vllm_args.tensor-parallel-size", tensor_parallel) + tp_key = {"sglang": "tp-size", "trtllm": "tp_size"}.get( + resolved_engine, "tensor-parallel-size" + ) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.args.{tp_key}", tensor_parallel + ) if replicas is not None: config.project.set_config("rhaiis.deploy.replicas", replicas) if storage_source: @@ -169,9 +188,10 @@ def _print_dry_run( workload_cfg = runtime_config.get_workload(workload_key) deploy_cfg = runtime_config.get_deploy_config() accelerator = runtime_config.get_accelerator() - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload_cfg) + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_args = runtime_config.merge_engine_args(engine_defaults, model_cfg, workload_cfg, engine) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) if not deployment_name: @@ -182,9 +202,10 @@ def _print_dry_run( click.echo(f" Workload: {workload_key}") click.echo(f" Namespace: {namespace}") click.echo(f" Deployment: {deployment_name}") + click.echo(f" Engine: {engine}") click.echo(f" Accelerator: {accelerator}") - click.echo(f" Image: {vllm_image}") - click.echo(f" vLLM args: {vllm_args}") + click.echo(f" Image: {serving_image}") + click.echo(f" Engine args: {engine_args}") click.echo(f" Env vars: {env_vars}") click.echo(f" Replicas: {deploy_cfg.get('replicas', 1)}") click.echo( diff --git a/projects/rhaiis/orchestration/config.d/models.yaml b/projects/rhaiis/orchestration/config.d/models.yaml index 806ce0442..1a839f919 100644 --- a/projects/rhaiis/orchestration/config.d/models.yaml +++ b/projects/rhaiis/orchestration/config.d/models.yaml @@ -2,46 +2,46 @@ qwen3-0_6b: name: Qwen3-0.6B hf_model_id: Qwen/Qwen3-0.6B - vllm_args: {} + engine_args: {} # Llama-4 Scout (MoE) llama-4-scout: name: Llama-4-Scout-17B-16E-Instruct hf_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-4-scout-fp8: name: Llama-4-Scout-17B-16E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 2 kv-cache-dtype: fp8 llama-4-scout-int4: name: Llama-4-Scout-17B-16E-Instruct-INT4 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 2 # Llama-4 Maverick (MoE) llama-4-maverick: name: Llama-4-Maverick-17B-128E-Instruct hf_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 8 llama-4-maverick-fp8: name: Llama-4-Maverick-17B-128E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 8 kv-cache-dtype: fp8 llama-4-maverick-w4a16: name: Llama-4-Maverick-17B-128E-Instruct-W4A16 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 enable-expert-parallel: true @@ -66,25 +66,25 @@ mistral-2503-fp8: llama-3-3-70b: name: Llama-3.3-70B-Instruct hf_model_id: meta-llama/Llama-3.3-70B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-fp8: name: Llama-3.3-70B-Instruct-FP8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w8a8: name: Llama-3.3-70B-Instruct-W8A8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w8a8 - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w4a16: name: Llama-3.3-70B-Instruct-W4A16 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 4 # Granite 3.1 8B @@ -125,7 +125,7 @@ qwen25-7b-w4a16: qwen3-n-80b-a3b-instruct: name: Qwen3-Next-80B-A3B-Instruct hf_model_id: Qwen/Qwen3-Next-80B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-30b-a3b-instruct: @@ -135,13 +135,13 @@ qwen3-30b-a3b-instruct: qwen3-30b-a3b: name: Qwen3-30B-A3B hf_model_id: Qwen/Qwen3-30B-A3B - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-235b-instruct: name: Qwen3-235B-A22B-Instruct hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507 - vllm_args: + engine_args: tensor-parallel-size: 4 env_vars: VLLM_ROCM_USE_AITER: "0" @@ -149,7 +149,7 @@ qwen3-235b-instruct: qwen3-235b-instruct-fp8: name: Qwen3-235B-A22B-Instruct-FP8 hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true kv-cache-dtype: fp8 @@ -157,20 +157,20 @@ qwen3-235b-instruct-fp8: qwen3-235b-fp8-dynamic: name: Qwen3-235B-A22B-FP8-dynamic hf_model_id: RedHatAI/Qwen3-235B-A22B-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true qwen3-vl-235b-fp8: name: Qwen3-VL-235B-A22B-FP8 hf_model_id: Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-vl-30b-a3b: name: Qwen3-VL-30B-A3B-Instruct hf_model_id: Qwen/Qwen3-VL-30B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 # GPT-OSS @@ -200,7 +200,7 @@ mixtral-8x7b: nemotron-3-1-70b: name: Nemotron-70B-Instruct hf_model_id: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF - vllm_args: + engine_args: tensor-parallel-size: 2 nemotron-3-1-70b-fp8: @@ -224,25 +224,25 @@ gemma-9b-w8a16: deepseek-v4-pro: name: DeepSeek-V4-Pro hf_model_id: deepseek-ai/DeepSeek-V4-Pro - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528: name: DeepSeek-R1-0528 hf_model_id: deepseek-ai/DeepSeek-R1-0528 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528-w4a16: name: DeepSeek-R1-0528-W4A16 hf_model_id: RedHatAI/DeepSeek-R1-0528-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-v3-2: name: DeepSeek-V3.2 hf_model_id: deepseek-ai/DeepSeek-V3.2 - vllm_args: + engine_args: tensor-parallel-size: 8 # Mistral 7B @@ -258,30 +258,30 @@ nemotron3nano-30b: nemotron3nano-30b-fp8: name: Nemotron-3-Nano-30B-A3B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 nemotron3super-120b-fp8: name: Nemotron-3-Super-120B-A12B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 # Ministral ministral-3-14b-reasoning: name: Ministral-3-14B-Reasoning hf_model_id: mistralai/Ministral-3-14B-Reasoning-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp16: name: Ministral-3-14B-Instruct hf_model_id: mistralai/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp8: name: Ministral-3-14B-FP8 hf_model_id: RedHatAI/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 90faeab5b..ae95755d3 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -11,6 +11,33 @@ images: nvidia: quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2-1773886296 amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5-1766067105 +engine: vllm + +engines: + vllm: + images: + nvidia: vllm/vllm-openai:v0.24.0 + amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5 + port: 8080 + args: + gpu-memory-utilization: 0.92 + trust-remote-code: true + no-enable-log-requests: true + no-enable-prefix-caching: true + uvicorn-log-level: debug + sglang: + images: + nvidia: lmsysorg/sglang:v0.5.11 + amd: lmsysorg/sglang:v0.5.4.post2-rocm700-mi30x + port: 8080 + args: {} + trtllm: + images: + nvidia: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc13 + port: 8888 + args: {} + trtllm_config: {} + deploy: replicas: 1 cpu_request: "4" @@ -22,13 +49,6 @@ deploy: ready_timeout: 3600 health_check_timeout: 120 -vllm_args: - gpu-memory-utilization: 0.92 - trust-remote-code: true - no-enable-log-requests: true - no-enable-prefix-caching: true - uvicorn-log-level: debug - env_vars: {} accelerator_env_vars: diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py index b500188b3..fcaddc5f6 100644 --- a/projects/rhaiis/orchestration/notifications.py +++ b/projects/rhaiis/orchestration/notifications.py @@ -25,11 +25,20 @@ def _send_alert( accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") - vllm_args = runtime_config.merge_vllm_args( - runtime_config.get_vllm_defaults(), + engine = runtime_config.get_engine() + engine_args = runtime_config.merge_engine_args( + runtime_config.get_engine_args(engine), model_cfg, runtime_config.get_workload(workload_keys[0]), + engine, ) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or "" + ) + dp = engine_args.get("data-parallel-size") or engine_args.get("dp-size") or "" send_failure_notification( error=error_message, @@ -38,8 +47,8 @@ def _send_alert( job_id=os.environ.get("FJOB_NAME", ""), slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), notification_vault="psap-forge-notifications", - tp=str(vllm_args.get("tensor-parallel-size", "")), - dp=str(vllm_args.get("data-parallel-size", "")), + tp=str(tp), + dp=str(dp), version=_cfg.project.get_config("tests.rhaiis.version", ""), workload_keys=workload_keys, cluster=cluster_tag, diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index 86c620f6b..a2ed40b80 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -75,3 +75,13 @@ nvidia: amd: rhaiis.accelerator: amd + +# Engine +vllm: + rhaiis.engine: vllm + +sglang: + rhaiis.engine: sglang + +trtllm: + rhaiis.engine: trtllm diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 1abbc2492..3f898fa81 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -39,12 +39,30 @@ def get_benchmark_config() -> dict: return dict(config.project.get_config("benchmarks.guidellm")) -def get_vllm_image(accelerator: str) -> str: - return config.project.get_config(f"rhaiis.images.{accelerator}") +def get_engine() -> str: + return config.project.get_config("rhaiis.engine", "vllm") -def get_vllm_defaults() -> dict: - return dict(config.project.get_config("rhaiis.vllm_args")) +def get_serving_image(accelerator: str, engine: str | None = None) -> str: + engine = engine or get_engine() + try: + return config.project.get_config(f"rhaiis.engines.{engine}.images.{accelerator}") + except Exception: + return config.project.get_config(f"rhaiis.images.{accelerator}") + + +def get_engine_args(engine: str | None = None) -> dict: + engine = engine or get_engine() + return dict(config.project.get_config(f"rhaiis.engines.{engine}.args", {})) + + +def get_engine_port(engine: str | None = None) -> int: + engine = engine or get_engine() + return int(config.project.get_config(f"rhaiis.engines.{engine}.port", 8080)) + + +def get_trtllm_config() -> dict: + return dict(config.project.get_config("rhaiis.engines.trtllm.trtllm_config", {})) def get_model(model_key: str) -> dict: @@ -71,15 +89,62 @@ def get_profiler_config() -> dict: return dict(config.project.get_config("rhaiis.profiler", {}) or {}) -def merge_vllm_args( +_COMMON_ARG_TRANSLATIONS: dict[str, dict[str, str]] = { + "sglang": { + "tensor-parallel-size": "tp-size", + "data-parallel-size": "dp-size", + }, + "trtllm": { + "tensor-parallel-size": "tp_size", + "data-parallel-size": "dp_size", + }, +} + + +def _translate_args(args: dict, engine: str) -> dict: + """Translate vLLM-style engine_args to another engine's arg naming. + + Only shared args (TP, DP) are translated; vLLM-specific args are dropped. + """ + mapping = _COMMON_ARG_TRANSLATIONS.get(engine, {}) + translated = {} + for key, val in args.items(): + if key in mapping: + translated[mapping[key]] = val + # Drop vLLM-only args that have no equivalent + return translated + + +def merge_engine_args( overrides: dict, model: dict, workload: dict, + engine: str | None = None, ) -> dict: - merged = dict(model.get("vllm_args", {})) - merged.update(workload.get("vllm_args", {})) - merged.update(overrides) - return merged + engine = engine or get_engine() + engine_key = f"{engine}_args" + + # Use engine-specific block if present, otherwise fall back to engine_args + # and auto-translate common args when the engine isn't vLLM + model_args = model.get(engine_key) if engine != "vllm" else None + if model_args is None: + base = dict(model.get("engine_args", {})) + if engine != "vllm": + base = _translate_args(base, engine) + else: + base = dict(model_args) + + wl_args = workload.get(engine_key) if engine != "vllm" else None + if wl_args is None: + wl = dict(workload.get("engine_args", {})) + if engine != "vllm": + wl = _translate_args(wl, engine) + else: + wl = dict(wl_args) + + base.update(wl) + base.update(overrides) + return base def merge_env_vars(accelerator: str, model: dict) -> dict: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index bef05895d..951793a28 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -100,10 +100,14 @@ def _run_test( else: deployment_name = base_name - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_port = runtime_config.get_engine_port(engine) first_workload = runtime_config.get_workload(workload_keys[0]) - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + engine_args = runtime_config.merge_engine_args( + engine_defaults, model_cfg, first_workload, engine + ) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) from projects.core.library import config as _cfg @@ -160,7 +164,7 @@ def _run_test( run_standalone_analysis( model_cfg, accelerator_key, - vllm_args, + engine_args, run_uuid=run_uuid, restrict_profiles=workload_keys or None, ) @@ -174,17 +178,22 @@ def _run_test( try: isvc_labels = {"opendatahub.io/dashboard": "true"} - if profiler_enabled: + if profiler_enabled and engine == "vllm": isvc_labels["vllm-profiler/enabled"] = "true" + elif profiler_enabled and engine != "vllm": + logger.warning("Profiler is only supported with vLLM engine, skipping profiler") + profiler_enabled = False logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) - deploy_kserve_isvc( + deploy_kwargs = dict( deployment_name=deployment_name, namespace=namespace, model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, + serving_image=serving_image, accelerator=accelerator, - vllm_args=vllm_args, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, env_vars=env_vars, replicas=deploy_cfg.get("replicas", 1), cpu_request=deploy_cfg.get("cpu_request", "4"), @@ -195,6 +204,9 @@ def _run_test( service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) + if engine == "trtllm": + deploy_kwargs["trtllm_config"] = runtime_config.get_trtllm_config() or None + deploy_kserve_isvc(**deploy_kwargs) logger.info("Waiting for InferenceService to be ready") wait_isvc_ready( @@ -204,7 +216,9 @@ def _run_test( health_check_timeout=deploy_cfg.get("health_check_timeout", 120), ) - endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" + endpoint_url = ( + f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:{engine_port}" + ) warmup_enabled = ( config.project.get_config("tests.rhaiis.warmup", True) @@ -239,7 +253,7 @@ def _run_test( if profiler_enabled: try: - _upload_profiler_traces(model_cfg, gpu_type, vllm_args, profiler_cfg) + _upload_profiler_traces(model_cfg, gpu_type, engine_args, profiler_cfg) except Exception: logger.exception("Profiler trace upload failed") _warnings.append("Profiler trace upload failed") @@ -253,8 +267,8 @@ def _run_test( accelerator=accelerator, accelerator_key=accelerator_key, gpu_type=gpu_type, - vllm_image=vllm_image, - vllm_args=vllm_args, + serving_image=serving_image, + engine_args=engine_args, benchmark_cfg=benchmark_cfg, deployment_name=deployment_name, namespace=namespace, @@ -274,8 +288,8 @@ def _run_test( ",".join(workload_keys), model_cfg, accelerator, - vllm_image, - vllm_args, + serving_image, + engine_args, benchmark_cfg, first_rates, first_max_seconds, @@ -318,8 +332,8 @@ def _run_workload_benchmark( accelerator: str, accelerator_key: str, gpu_type: str, - vllm_image: str, - vllm_args: dict, + serving_image: str, + engine_args: dict, benchmark_cfg: dict, deployment_name: str, namespace: str, @@ -353,10 +367,10 @@ def _run_workload_benchmark( model_key, workload_key, accelerator, - vllm_args, + engine_args, hf_model_id=model_cfg["hf_model_id"], version=version, - vllm_image=vllm_image, + serving_image=serving_image, cluster_tag=cluster_tag, accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, @@ -370,7 +384,7 @@ def _run_workload_benchmark( run_standalone_analysis( model_cfg, accelerator_key, - vllm_args, + engine_args, run_uuid=run_uuid, restrict_profiles=[workload_key], ) @@ -406,22 +420,28 @@ def _create_test_labels( model_key: str, workload_key: str, accelerator: str, - vllm_args: dict, + engine_args: dict, *, hf_model_id: str = "", version: str = "", - vllm_image: str = "", + serving_image: str = "", cluster_tag: str = "", accelerator_chip: str = "", run_uuid: str = "", ) -> None: - _, image_tag = runtime_config.split_image_tag(vllm_image) if vllm_image else ("", "") - runtime_args = "; ".join(f"{k}: {v}" for k, v in vllm_args.items()) + _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") + runtime_args = "; ".join(f"{k}: {v}" for k, v in engine_args.items()) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) labels = { "model_key": model_key, "workload_key": workload_key, "accelerator": accelerator_chip or accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), + "tensor_parallel_size": str(tp), "hf_model_id": hf_model_id, "version": version, "image_tag": image_tag, @@ -438,8 +458,8 @@ def _set_mlflow_metadata( workload_key: str, model_cfg: dict, accelerator: str, - vllm_image: str, - vllm_args: dict, + serving_image: str, + engine_args: dict, benchmark_cfg: dict, rates: list[int], max_seconds: int, @@ -448,18 +468,24 @@ def _set_mlflow_metadata( ) -> None: from projects.core.library import config - image_name, image_tag = runtime_config.split_image_tag(vllm_image) + image_name, image_tag = runtime_config.split_image_tag(serving_image) guidellm_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") benchmark_args = benchmark_cfg.get("args", {}) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) tags = { "project": "rhaiis", "model_key": model_key, "hf_model_id": model_cfg["hf_model_id"], "accelerator": accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), - "vllm_image": vllm_image, - "vllm_version": image_tag, + "tensor_parallel_size": str(tp), + "serving_image": serving_image, + "serving_version": image_tag, "workload_key": workload_key, "rates": ",".join(str(r) for r in rates), "max_seconds": str(max_seconds), @@ -521,9 +547,10 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) model_cfg = runtime_config.get_model(model_key) accelerator = runtime_config.get_accelerator() - vllm_defaults = runtime_config.get_vllm_defaults() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) first_workload = runtime_config.get_workload(workload_keys[0]) - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, first_workload) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) from projects.rhaiis.orchestration.analysis import run_regression_check @@ -534,7 +561,7 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) model_cfg, accelerator, run_uuid="", - vllm_args=vllm_args, + engine_args=ea, ) @@ -717,7 +744,7 @@ def _infer_profile_labels_from_traces(trace_files: list) -> list[str]: def _upload_profiler_traces( model_cfg: dict, accelerator: str, - vllm_args: dict, + engine_args: dict, profiler_cfg: dict, ) -> None: from pathlib import Path @@ -757,7 +784,12 @@ def _upload_profiler_traces( traces_dir, model_name=model_cfg.get("hf_model_id", ""), accelerator=accelerator, - tp_size=int(vllm_args.get("tensor-parallel-size", 1)), + tp_size=int( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ), version=version, profile_labels=profile_labels, s3_bucket=s3_cfg.get("bucket", ""), diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index e8908595a..c5d4e0b43 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -15,9 +15,12 @@ def run( deployment_name: str, namespace: str, model_id: str, - vllm_image: str, + serving_image: str, accelerator: str = "nvidia", - vllm_args: dict | None = None, + engine: str = "vllm", + engine_args: dict | None = None, + engine_port: int = 8080, + trtllm_config: dict | None = None, env_vars: dict | None = None, replicas: int = 1, cpu_request: str = "4", @@ -37,8 +40,8 @@ def prepare_args(args, context): for key, value in (args.env_vars or {}).items(): context.env_vars_list.append({"name": key, "value": str(value)}) - vllm_args = args.vllm_args or {} - tp_size = vllm_args.get("tensor-parallel-size", 1) + ea = args.engine_args or {} + tp_size = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 context.gpu_count = int(tp_size) if args.storage_pvc: @@ -48,7 +51,7 @@ def prepare_args(args, context): context.use_pvc = False context.pvc_name = "" - return f"GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" + return f"engine={args.engine}, GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" @task diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 index 5ca599383..5d728620f 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 @@ -3,41 +3,66 @@ kind: InferenceService metadata: annotations: serving.kserve.io/deploymentMode: RawDeployment - serving.kserve.io/enable-prometheus-scraping: "true" storage.kserve.io/readonly: "false" +{%- if args.engine != "sglang" %} + serving.kserve.io/enable-prometheus-scraping: "true" prometheus.io/scrape: "true" prometheus.io/path: /metrics - prometheus.io/port: "8080" + prometheus.io/port: "{{ args.engine_port }}" +{%- endif %} {%- if args.labels %} labels: {%- for key, val in args.labels.items() %} {{ key }}: "{{ val }}" {%- endfor %} +{%- if args.engine == "sglang" or args.engine == "trtllm" %} + monitoring.opendatahub.io/scrape: "false" +{%- endif %} {%- endif %} name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: predictor: +{%- if args.engine == "trtllm" %} + terminationGracePeriodSeconds: 120 +{%- endif %} minReplicas: {{ args.replicas }} maxReplicas: {{ args.replicas }} model: modelFormat: +{%- if args.engine == "sglang" %} + name: sglang +{%- elif args.engine == "trtllm" %} + name: trtllm +{%- else %} name: vLLM +{%- endif %} runtime: {{ args.deployment_name }} resources: {%- if args.accelerator == "nvidia" %} limits: nvidia.com/gpu: "{{ ctx.gpu_count }}" +{%- if args.engine == "trtllm" %} + memory: "{{ args.memory_request }}" + ephemeral-storage: "50Gi" +{%- endif %} requests: nvidia.com/gpu: "{{ ctx.gpu_count }}" +{%- if args.engine == "trtllm" %} + cpu: "{{ args.cpu_request }}" + memory: "{{ args.memory_request }}" + ephemeral-storage: "50Gi" +{%- endif %} {%- elif args.accelerator == "amd" %} limits: amd.com/gpu: "{{ ctx.gpu_count }}" requests: amd.com/gpu: "{{ ctx.gpu_count }}" {%- endif %} +{%- if not (args.accelerator == "nvidia" and args.engine == "trtllm") %} cpu: "{{ args.cpu_request }}" memory: "{{ args.memory_request }}" +{%- endif %} {%- if args.storage_source == "hf" and ctx.use_pvc %} storageUri: pvc://{{ ctx.pvc_name }}/ {%- elif args.storage_source != "hf" %} diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index 42d53cd73..2b077c658 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: ServingRuntime metadata: annotations: - opendatahub.io/template-display-name: ServingRuntime for vLLM | RHAIIS + opendatahub.io/template-display-name: ServingRuntime for {{ args.engine }} | RHAIIS labels: opendatahub.io/dashboard: "true" name: {{ args.deployment_name }} @@ -11,17 +11,117 @@ spec: {%- if args.image_pull_secret %} imagePullSecrets: - name: {{ args.image_pull_secret }} +{%- if args.engine == "trtllm" %} + - name: ngc-secret +{%- endif %} +{%- elif args.engine == "trtllm" %} + imagePullSecrets: + - name: ngc-secret {%- endif %} builtInAdapter: modelLoadingTimeoutMillis: 300000 multiModel: false supportedModelFormats: - autoSelect: true +{%- if args.engine == "sglang" %} + name: sglang +{%- elif args.engine == "trtllm" %} + name: trtllm +{%- else %} name: vLLM +{%- endif %} containers: +{%- if args.engine == "trtllm" %} + - name: kserve-container + image: "{{ args.serving_image }}" + command: + - /bin/bash + - -c + - | + set -ex + if [ ! -d "/mnt/models/hub/models--{{ args.model_id | replace('/', '--') }}/snapshots" ]; then + hf download {{ args.model_id }} + fi + python3 -c " + from flashinfer.norm import get_norm_module + get_norm_module() + print('FlashInfer norm kernel pre-compiled successfully') + " +{%- if args.trtllm_config %} + cat > /tmp/trtllm-config.yml << EOF + {{ args.trtllm_config | indent(10) }} + EOF +{%- endif %} + trtllm-serve {{ args.model_id }} \ + --host 0.0.0.0 \ + --port={{ args.engine_port }} \ +{%- for key, val in (args.engine_args or {}).items() %} + {%- if key != "trtllm-config" %} + {%- if val is boolean %} + {%- if val %} + --{{ key }} \ + {%- endif %} + {%- else %} + --{{ key }}={{ val }} \ + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if args.trtllm_config %} + --extra_llm_api_options=/tmp/trtllm-config.yml +{%- endif %} + + env: + - name: HF_HUB_OFFLINE + value: "0" + - name: HF_HOME + value: /mnt/models + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: storage-config + key: HF_TOKEN + - name: TRTLLM_ENABLE_PDL + value: "1" + - name: HOME + value: /tmp + - name: XDG_CACHE_HOME + value: /tmp/.cache + - name: FLASHINFER_WORKSPACE_BASE + value: /mnt/models +{%- for env_item in ctx.env_vars_list %} + - name: {{ env_item.name }} + value: "{{ env_item.value }}" +{%- endfor %} + securityContext: + runAsUser: 0 + ports: + - containerPort: {{ args.engine_port }} + protocol: TCP + volumeMounts: + - name: shared-memory + mountPath: /dev/shm + volumes: + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: 128Gi +{%- else %} - name: kserve-container - image: "{{ args.vllm_image }}" + image: "{{ args.serving_image }}" command: +{%- if args.engine == "sglang" %} + - python3 + - -m + - sglang.launch_server + args: +{%- if args.storage_source == "hf" %} + - --model-path={{ args.model_id }} +{%- else %} + - --model-path=/mnt/models +{%- endif %} + - --port=8080 + - --host=0.0.0.0 +{%- else %} - python3 - -m - vllm.entrypoints.openai.api_server @@ -33,7 +133,8 @@ spec: - --served-model-name={{ args.model_id }} {%- endif %} - --port=8080 -{%- for key, val in (args.vllm_args or {}).items() %} +{%- endif %} +{%- for key, val in (args.engine_args or {}).items() %} {%- if val is boolean %} {%- if val %} - --{{ key }} @@ -43,8 +144,14 @@ spec: {%- endif %} {%- endfor %} env: - - name: USER - value: "vllm" + - name: XDG_CACHE_HOME + value: /tmp/.cache + - name: XDG_CONFIG_HOME + value: /tmp/.config +{%- if ctx.gpu_count > 1 %} + - name: NCCL_DEBUG + value: "WARN" +{%- endif %} {%- if args.storage_source == "hf" %} - name: HF_HUB_OFFLINE value: "0" @@ -52,8 +159,10 @@ spec: value: /mnt/models - name: HF_HOME value: /mnt/models +{%- if args.engine != "sglang" %} - name: VLLM_CACHE_DIR value: /mnt/models/.cache/vllm +{%- endif %} - name: HF_DATASETS_CACHE value: /mnt/models/.cache/huggingface/datasets - name: HF_TOKEN @@ -61,13 +170,18 @@ spec: secretKeyRef: name: storage-config key: HF_TOKEN +{%- else %} + - name: HOME + value: /tmp {%- endif %} {%- for env_item in ctx.env_vars_list %} - name: {{ env_item.name }} value: "{{ env_item.value }}" {%- endfor %} + securityContext: + runAsUser: 0 ports: - - containerPort: 8080 + - containerPort: {{ args.engine_port }} protocol: TCP {%- if ctx.gpu_count > 1 %} volumeMounts: @@ -77,5 +191,6 @@ spec: - name: shared-memory emptyDir: medium: Memory - sizeLimit: 8Gi + sizeLimit: 64Gi +{%- endif %} {%- endif %} From cf28237922e1c4dae832838eb1467a527927b27b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 13:56:54 -0400 Subject: [PATCH 0218/1072] fix: remove runAsUser:0 from vLLM/SGLang ServingRuntime OpenShift SCC rejects runAsUser:0 when the service account lacks the anyuid SCC. vLLM and SGLang don't require root, so drop it. TRT-LLM retains runAsUser:0 as it needs root for flashinfer compilation. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index 2b077c658..fbf63f759 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -178,8 +178,6 @@ spec: - name: {{ env_item.name }} value: "{{ env_item.value }}" {%- endfor %} - securityContext: - runAsUser: 0 ports: - containerPort: {{ args.engine_port }} protocol: TCP From ea8f55569e479adbd4d15f6b4cf64f90d613465a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 14:47:33 -0400 Subject: [PATCH 0219/1072] fix: add USER env var back for vLLM/SGLang ServingRuntime PyTorch's torch._inductor calls getpass.getuser() which falls back to pwd.getpwuid(). OpenShift assigns arbitrary UIDs not in /etc/passwd, causing a KeyError. Setting USER env var avoids the passwd lookup. Co-authored-by: Cursor --- .../toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index fbf63f759..c4bec6e4c 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -144,6 +144,8 @@ spec: {%- endif %} {%- endfor %} env: + - name: USER + value: "{{ args.engine }}" - name: XDG_CACHE_HOME value: /tmp/.cache - name: XDG_CONFIG_HOME From fa0dab0db93b823f610eeb752c4a4a4055d59421 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 15:09:53 -0400 Subject: [PATCH 0220/1072] refactor: replace image_pull_secret string with image_pull_secrets list Remove hardcoded ngc-secret from ServingRuntime template. Users now configure all pull secrets through rhaiis.deploy.image_pull_secrets as a list, including any NGC secrets needed for TRT-LLM. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- projects/rhaiis/orchestration/test_phase.py | 2 +- projects/rhaiis/toolbox/deploy_kserve_isvc/main.py | 2 +- .../templates/servingruntime.yaml.j2 | 12 ++++-------- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index ae95755d3..2b0d89124 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -44,7 +44,7 @@ deploy: memory_request: "16Gi" storage_source: hf storage_pvc: model-pvc - image_pull_secret: "" + image_pull_secrets: [] service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 951793a28..32747fa84 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -200,7 +200,7 @@ def _run_test( memory_request=deploy_cfg.get("memory_request", "16Gi"), storage_source=deploy_cfg.get("storage_source", "hf"), storage_pvc=deploy_cfg.get("storage_pvc", ""), - image_pull_secret=deploy_cfg.get("image_pull_secret", ""), + image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index c5d4e0b43..1a8b0e4a8 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -27,7 +27,7 @@ def run( memory_request: str = "16Gi", storage_source: str = "hf", storage_pvc: str = "", - image_pull_secret: str = "", + image_pull_secrets: list | None = None, service_account_name: str = "", labels: dict | None = None, ): diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 index c4bec6e4c..e5f8d80f6 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 @@ -8,15 +8,11 @@ metadata: name: {{ args.deployment_name }} namespace: {{ args.namespace }} spec: -{%- if args.image_pull_secret %} +{%- if args.image_pull_secrets %} imagePullSecrets: - - name: {{ args.image_pull_secret }} -{%- if args.engine == "trtllm" %} - - name: ngc-secret -{%- endif %} -{%- elif args.engine == "trtllm" %} - imagePullSecrets: - - name: ngc-secret +{%- for secret in args.image_pull_secrets %} + - name: {{ secret }} +{%- endfor %} {%- endif %} builtInAdapter: modelLoadingTimeoutMillis: 300000 From 6c1c3970700bd24a9f8011c8d1733c9d1763ab3c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 15:17:26 -0400 Subject: [PATCH 0221/1072] refactor: move manifest building to orchestration layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace Jinja2 templates with Python dict construction in a new manifests.py module. The deploy_kserve_isvc script is now generic — it accepts pre-built manifest dicts and applies them, with no engine awareness. This follows the pattern established by llm_d. Co-authored-by: Cursor --- projects/rhaiis/orchestration/manifests.py | 376 ++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 34 +- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 73 ++-- .../templates/inferenceservice.yaml.j2 | 73 ---- .../templates/servingruntime.yaml.j2 | 192 --------- 5 files changed, 433 insertions(+), 315 deletions(-) create mode 100644 projects/rhaiis/orchestration/manifests.py delete mode 100644 projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 delete mode 100644 projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py new file mode 100644 index 000000000..c181288a4 --- /dev/null +++ b/projects/rhaiis/orchestration/manifests.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def build_servingruntime( + *, + deployment_name: str, + namespace: str, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + image_pull_secrets: list[str] | None = None, + env_vars: dict | None = None, + trtllm_config: dict | None = None, +) -> dict[str, Any]: + """Build a KServe ServingRuntime manifest dict.""" + env_vars_list = [{"name": k, "value": str(v)} for k, v in (env_vars or {}).items()] + + manifest: dict[str, Any] = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "annotations": { + "opendatahub.io/template-display-name": f"ServingRuntime for {engine} | RHAIIS", + }, + "labels": {"opendatahub.io/dashboard": "true"}, + "name": deployment_name, + "namespace": namespace, + }, + "spec": { + "builtInAdapter": {"modelLoadingTimeoutMillis": 300000}, + "multiModel": False, + "supportedModelFormats": [ + {"autoSelect": True, "name": _model_format_name(engine)}, + ], + }, + } + + if image_pull_secrets: + manifest["spec"]["imagePullSecrets"] = [{"name": s} for s in image_pull_secrets] + + if engine == "trtllm": + container = _build_trtllm_container( + model_id=model_id, + serving_image=serving_image, + engine_args=engine_args, + engine_port=engine_port, + env_vars_list=env_vars_list, + trtllm_config=trtllm_config, + ) + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "128Gi"}}, + ] + else: + container = _build_vllm_sglang_container( + model_id=model_id, + serving_image=serving_image, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, + storage_source=storage_source, + gpu_count=gpu_count, + env_vars_list=env_vars_list, + ) + if gpu_count > 1: + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "64Gi"}}, + ] + + manifest["spec"]["containers"] = [container] + return manifest + + +def build_inferenceservice( + *, + deployment_name: str, + namespace: str, + engine: str, + engine_port: int, + accelerator: str, + gpu_count: int, + replicas: int, + cpu_request: str, + memory_request: str, + storage_source: str, + storage_pvc: str, + model_id: str, + service_account_name: str = "", + labels: dict | None = None, +) -> dict[str, Any]: + """Build a KServe InferenceService manifest dict.""" + annotations: dict[str, str] = { + "serving.kserve.io/deploymentMode": "RawDeployment", + "storage.kserve.io/readonly": "false", + } + if engine != "sglang": + annotations.update( + { + "serving.kserve.io/enable-prometheus-scraping": "true", + "prometheus.io/scrape": "true", + "prometheus.io/path": "/metrics", + "prometheus.io/port": str(engine_port), + } + ) + + metadata: dict[str, Any] = { + "annotations": annotations, + "name": deployment_name, + "namespace": namespace, + } + + if labels: + meta_labels = dict(labels) + if engine in ("sglang", "trtllm"): + meta_labels["monitoring.opendatahub.io/scrape"] = "false" + metadata["labels"] = meta_labels + + resources = _build_resources( + accelerator=accelerator, + gpu_count=gpu_count, + cpu_request=cpu_request, + memory_request=memory_request, + engine=engine, + ) + + model_spec: dict[str, Any] = { + "modelFormat": {"name": _model_format_name(engine)}, + "runtime": deployment_name, + "resources": resources, + } + + if storage_source == "hf" and storage_pvc: + model_spec["storageUri"] = f"pvc://{storage_pvc}/" + elif storage_source != "hf": + model_spec["storageUri"] = f"{storage_source}://{model_id}" + + predictor: dict[str, Any] = { + "minReplicas": replicas, + "maxReplicas": replicas, + "model": model_spec, + } + + if engine == "trtllm": + predictor["terminationGracePeriodSeconds"] = 120 + + if service_account_name: + predictor["serviceAccountName"] = service_account_name + + return { + "apiVersion": "serving.kserve.io/v1beta1", + "kind": "InferenceService", + "metadata": metadata, + "spec": {"predictor": predictor}, + } + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _model_format_name(engine: str) -> str: + if engine == "sglang": + return "sglang" + if engine == "trtllm": + return "trtllm" + return "vLLM" + + +def _build_resources( + *, + accelerator: str, + gpu_count: int, + cpu_request: str, + memory_request: str, + engine: str, +) -> dict[str, Any]: + resources: dict[str, Any] = {} + gpu_key = "nvidia.com/gpu" if accelerator == "nvidia" else "amd.com/gpu" + + if accelerator in ("nvidia", "amd"): + limits: dict[str, str] = {gpu_key: str(gpu_count)} + requests: dict[str, str] = {gpu_key: str(gpu_count)} + + if accelerator == "nvidia" and engine == "trtllm": + limits["memory"] = memory_request + limits["ephemeral-storage"] = "50Gi" + requests["cpu"] = cpu_request + requests["memory"] = memory_request + requests["ephemeral-storage"] = "50Gi" + + resources["limits"] = limits + resources["requests"] = requests + + if not (accelerator == "nvidia" and engine == "trtllm"): + resources.setdefault("requests", {}) + resources["requests"]["cpu"] = cpu_request + resources["requests"]["memory"] = memory_request + + return resources + + +def _build_vllm_sglang_container( + *, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + env_vars_list: list[dict], +) -> dict[str, Any]: + if engine == "sglang": + command = ["python3", "-m", "sglang.launch_server"] + if storage_source == "hf": + args = [f"--model-path={model_id}", "--port=8080", "--host=0.0.0.0"] + else: + args = ["--model-path=/mnt/models", "--port=8080", "--host=0.0.0.0"] + else: + command = ["python3", "-m", "vllm.entrypoints.openai.api_server"] + if storage_source == "hf": + args = [f"--model={model_id}", "--port=8080"] + else: + args = ["--model=/mnt/models", f"--served-model-name={model_id}", "--port=8080"] + + for key, val in (engine_args or {}).items(): + if isinstance(val, bool): + if val: + args.append(f"--{key}") + else: + args.append(f"--{key}={val}") + + env: list[dict[str, Any]] = [ + {"name": "USER", "value": engine}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "XDG_CONFIG_HOME", "value": "/tmp/.config"}, + ] + + if gpu_count > 1: + env.append({"name": "NCCL_DEBUG", "value": "WARN"}) + + if storage_source == "hf": + env.extend( + [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HOME", "value": "/mnt/models"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + ] + ) + if engine != "sglang": + env.append({"name": "VLLM_CACHE_DIR", "value": "/mnt/models/.cache/vllm"}) + env.extend( + [ + {"name": "HF_DATASETS_CACHE", "value": "/mnt/models/.cache/huggingface/datasets"}, + { + "name": "HF_TOKEN", + "valueFrom": { + "secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}, + }, + }, + ] + ) + else: + env.append({"name": "HOME", "value": "/tmp"}) + + env.extend(env_vars_list) + + container: dict[str, Any] = { + "name": "kserve-container", + "image": serving_image, + "command": command, + "args": args, + "env": env, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + } + + if gpu_count > 1: + container["volumeMounts"] = [{"name": "shared-memory", "mountPath": "/dev/shm"}] + + return container + + +def _build_trtllm_launch_script( + model_id: str, + engine_port: int, + engine_args: dict, + trtllm_config: dict | None, +) -> str: + safe_model = model_id.replace("/", "--") + parts = [ + "set -ex", + f'if [ ! -d "/mnt/models/hub/models--{safe_model}/snapshots" ]; then', + f" hf download {model_id}", + "fi", + 'python3 -c "', + "from flashinfer.norm import get_norm_module", + "get_norm_module()", + "print('FlashInfer norm kernel pre-compiled successfully')", + '"', + ] + + if trtllm_config: + import yaml + + config_yaml = yaml.dump(trtllm_config, default_flow_style=False).rstrip() + parts.extend(["cat > /tmp/trtllm-config.yml << EOF", config_yaml, "EOF"]) + + cmd = [f"trtllm-serve {model_id} \\", " --host 0.0.0.0 \\", f" --port={engine_port}"] + + for key, val in (engine_args or {}).items(): + if key == "trtllm-config": + continue + if isinstance(val, bool): + if val: + cmd.append(f" --{key}") + else: + cmd.append(f" --{key}={val}") + + if trtllm_config: + cmd.append(" --extra_llm_api_options=/tmp/trtllm-config.yml") + + # Add line continuations between cmd parts (except last) + cmd_lines = [] + for i, line in enumerate(cmd): + if i < len(cmd) - 1 and not line.endswith("\\"): + cmd_lines.append(f"{line} \\") + else: + cmd_lines.append(line) + + parts.extend(cmd_lines) + return "\n".join(parts) + "\n" + + +def _build_trtllm_container( + *, + model_id: str, + serving_image: str, + engine_args: dict, + engine_port: int, + env_vars_list: list[dict], + trtllm_config: dict | None, +) -> dict[str, Any]: + script = _build_trtllm_launch_script(model_id, engine_port, engine_args, trtllm_config) + + env: list[dict[str, Any]] = [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + { + "name": "HF_TOKEN", + "valueFrom": {"secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}}, + }, + {"name": "TRTLLM_ENABLE_PDL", "value": "1"}, + {"name": "HOME", "value": "/tmp"}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "FLASHINFER_WORKSPACE_BASE", "value": "/mnt/models"}, + ] + env.extend(env_vars_list) + + return { + "name": "kserve-container", + "image": serving_image, + "command": ["/bin/bash", "-c", script], + "env": env, + "securityContext": {"runAsUser": 0}, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + "volumeMounts": [{"name": "shared-memory", "mountPath": "/dev/shm"}], + } diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 32747fa84..3ffbf7529 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -121,6 +121,10 @@ def _run_test( from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) + from projects.rhaiis.orchestration.manifests import ( + build_inferenceservice, + build_servingruntime, + ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready @@ -185,28 +189,46 @@ def _run_test( profiler_enabled = False logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) - deploy_kwargs = dict( + ea = engine_args or {} + gpu_count = int( + ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 + ) + + sr_manifest = build_servingruntime( deployment_name=deployment_name, namespace=namespace, model_id=model_cfg["hf_model_id"], serving_image=serving_image, - accelerator=accelerator, engine=engine, engine_args=engine_args, engine_port=engine_port, + storage_source=deploy_cfg.get("storage_source", "hf"), + gpu_count=gpu_count, + image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], env_vars=env_vars, + trtllm_config=runtime_config.get_trtllm_config() if engine == "trtllm" else None, + ) + isvc_manifest = build_inferenceservice( + deployment_name=deployment_name, + namespace=namespace, + engine=engine, + engine_port=engine_port, + accelerator=accelerator, + gpu_count=gpu_count, replicas=deploy_cfg.get("replicas", 1), cpu_request=deploy_cfg.get("cpu_request", "4"), memory_request=deploy_cfg.get("memory_request", "16Gi"), storage_source=deploy_cfg.get("storage_source", "hf"), storage_pvc=deploy_cfg.get("storage_pvc", ""), - image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], + model_id=model_cfg["hf_model_id"], service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) - if engine == "trtllm": - deploy_kwargs["trtllm_config"] = runtime_config.get_trtllm_config() or None - deploy_kserve_isvc(**deploy_kwargs) + deploy_kserve_isvc( + namespace=namespace, + servingruntime_manifest=sr_manifest, + inferenceservice_manifest=isvc_manifest, + ) logger.info("Waiting for InferenceService to be ready") wait_isvc_ready( diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index 1a8b0e4a8..4a0f7887e 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 +from __future__ import annotations + +from pathlib import Path + +import yaml + from projects.core.dsl import ( entrypoint, execute_tasks, task, - template, ) from projects.core.dsl.utils.k8s import oc, oc_resource_exists @@ -12,46 +17,18 @@ @entrypoint def run( *, - deployment_name: str, namespace: str, - model_id: str, - serving_image: str, - accelerator: str = "nvidia", - engine: str = "vllm", - engine_args: dict | None = None, - engine_port: int = 8080, - trtllm_config: dict | None = None, - env_vars: dict | None = None, - replicas: int = 1, - cpu_request: str = "4", - memory_request: str = "16Gi", - storage_source: str = "hf", - storage_pvc: str = "", - image_pull_secrets: list | None = None, - service_account_name: str = "", - labels: dict | None = None, + servingruntime_manifest: dict, + inferenceservice_manifest: dict, ): - return execute_tasks(locals()) - - -@task -def prepare_args(args, context): - context.env_vars_list = [] - for key, value in (args.env_vars or {}).items(): - context.env_vars_list.append({"name": key, "value": str(value)}) - - ea = args.engine_args or {} - tp_size = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 - context.gpu_count = int(tp_size) - - if args.storage_pvc: - context.use_pvc = True - context.pvc_name = args.storage_pvc - else: - context.use_pvc = False - context.pvc_name = "" + """Deploy a KServe InferenceService from pre-built manifests. - return f"engine={args.engine}, GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" + Args: + namespace: Target namespace for the deployment. + servingruntime_manifest: Pre-built ServingRuntime manifest dict. + inferenceservice_manifest: Pre-built InferenceService manifest dict. + """ + return execute_tasks(locals()) @task @@ -63,19 +40,27 @@ def ensure_namespace(args, context): @task -def render_and_apply_servingruntime(args, context): +def apply_servingruntime(args, context): output_path = args.artifact_dir / "servingruntime.yaml" - template.render_template_to_file("servingruntime.yaml.j2", output_path) + _write_manifest(args.servingruntime_manifest, output_path) oc("apply", "-f", str(output_path)) - return f"Applied ServingRuntime {args.deployment_name}" + name = args.servingruntime_manifest.get("metadata", {}).get("name", "") + return f"Applied ServingRuntime {name}" @task -def render_and_apply_inferenceservice(args, context): +def apply_inferenceservice(args, context): output_path = args.artifact_dir / "inferenceservice.yaml" - template.render_template_to_file("inferenceservice.yaml.j2", output_path) + _write_manifest(args.inferenceservice_manifest, output_path) oc("apply", "-f", str(output_path)) - return f"Applied InferenceService {args.deployment_name}" + name = args.inferenceservice_manifest.get("metadata", {}).get("name", "") + return f"Applied InferenceService {name}" + + +def _write_manifest(manifest: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) if __name__ == "__main__": diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 deleted file mode 100644 index 5d728620f..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - storage.kserve.io/readonly: "false" -{%- if args.engine != "sglang" %} - serving.kserve.io/enable-prometheus-scraping: "true" - prometheus.io/scrape: "true" - prometheus.io/path: /metrics - prometheus.io/port: "{{ args.engine_port }}" -{%- endif %} -{%- if args.labels %} - labels: -{%- for key, val in args.labels.items() %} - {{ key }}: "{{ val }}" -{%- endfor %} -{%- if args.engine == "sglang" or args.engine == "trtllm" %} - monitoring.opendatahub.io/scrape: "false" -{%- endif %} -{%- endif %} - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: - predictor: -{%- if args.engine == "trtllm" %} - terminationGracePeriodSeconds: 120 -{%- endif %} - minReplicas: {{ args.replicas }} - maxReplicas: {{ args.replicas }} - model: - modelFormat: -{%- if args.engine == "sglang" %} - name: sglang -{%- elif args.engine == "trtllm" %} - name: trtllm -{%- else %} - name: vLLM -{%- endif %} - runtime: {{ args.deployment_name }} - resources: -{%- if args.accelerator == "nvidia" %} - limits: - nvidia.com/gpu: "{{ ctx.gpu_count }}" -{%- if args.engine == "trtllm" %} - memory: "{{ args.memory_request }}" - ephemeral-storage: "50Gi" -{%- endif %} - requests: - nvidia.com/gpu: "{{ ctx.gpu_count }}" -{%- if args.engine == "trtllm" %} - cpu: "{{ args.cpu_request }}" - memory: "{{ args.memory_request }}" - ephemeral-storage: "50Gi" -{%- endif %} -{%- elif args.accelerator == "amd" %} - limits: - amd.com/gpu: "{{ ctx.gpu_count }}" - requests: - amd.com/gpu: "{{ ctx.gpu_count }}" -{%- endif %} -{%- if not (args.accelerator == "nvidia" and args.engine == "trtllm") %} - cpu: "{{ args.cpu_request }}" - memory: "{{ args.memory_request }}" -{%- endif %} -{%- if args.storage_source == "hf" and ctx.use_pvc %} - storageUri: pvc://{{ ctx.pvc_name }}/ -{%- elif args.storage_source != "hf" %} - storageUri: {{ args.storage_source }}://{{ args.model_id }} -{%- endif %} -{%- if args.service_account_name %} - serviceAccountName: {{ args.service_account_name }} -{%- endif %} diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 deleted file mode 100644 index e5f8d80f6..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ /dev/null @@ -1,192 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - annotations: - opendatahub.io/template-display-name: ServingRuntime for {{ args.engine }} | RHAIIS - labels: - opendatahub.io/dashboard: "true" - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: -{%- if args.image_pull_secrets %} - imagePullSecrets: -{%- for secret in args.image_pull_secrets %} - - name: {{ secret }} -{%- endfor %} -{%- endif %} - builtInAdapter: - modelLoadingTimeoutMillis: 300000 - multiModel: false - supportedModelFormats: - - autoSelect: true -{%- if args.engine == "sglang" %} - name: sglang -{%- elif args.engine == "trtllm" %} - name: trtllm -{%- else %} - name: vLLM -{%- endif %} - containers: -{%- if args.engine == "trtllm" %} - - name: kserve-container - image: "{{ args.serving_image }}" - command: - - /bin/bash - - -c - - | - set -ex - if [ ! -d "/mnt/models/hub/models--{{ args.model_id | replace('/', '--') }}/snapshots" ]; then - hf download {{ args.model_id }} - fi - python3 -c " - from flashinfer.norm import get_norm_module - get_norm_module() - print('FlashInfer norm kernel pre-compiled successfully') - " -{%- if args.trtllm_config %} - cat > /tmp/trtllm-config.yml << EOF - {{ args.trtllm_config | indent(10) }} - EOF -{%- endif %} - trtllm-serve {{ args.model_id }} \ - --host 0.0.0.0 \ - --port={{ args.engine_port }} \ -{%- for key, val in (args.engine_args or {}).items() %} - {%- if key != "trtllm-config" %} - {%- if val is boolean %} - {%- if val %} - --{{ key }} \ - {%- endif %} - {%- else %} - --{{ key }}={{ val }} \ - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if args.trtllm_config %} - --extra_llm_api_options=/tmp/trtllm-config.yml -{%- endif %} - - env: - - name: HF_HUB_OFFLINE - value: "0" - - name: HF_HOME - value: /mnt/models - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: storage-config - key: HF_TOKEN - - name: TRTLLM_ENABLE_PDL - value: "1" - - name: HOME - value: /tmp - - name: XDG_CACHE_HOME - value: /tmp/.cache - - name: FLASHINFER_WORKSPACE_BASE - value: /mnt/models -{%- for env_item in ctx.env_vars_list %} - - name: {{ env_item.name }} - value: "{{ env_item.value }}" -{%- endfor %} - securityContext: - runAsUser: 0 - ports: - - containerPort: {{ args.engine_port }} - protocol: TCP - volumeMounts: - - name: shared-memory - mountPath: /dev/shm - volumes: - - name: shared-memory - emptyDir: - medium: Memory - sizeLimit: 128Gi -{%- else %} - - name: kserve-container - image: "{{ args.serving_image }}" - command: -{%- if args.engine == "sglang" %} - - python3 - - -m - - sglang.launch_server - args: -{%- if args.storage_source == "hf" %} - - --model-path={{ args.model_id }} -{%- else %} - - --model-path=/mnt/models -{%- endif %} - - --port=8080 - - --host=0.0.0.0 -{%- else %} - - python3 - - -m - - vllm.entrypoints.openai.api_server - args: -{%- if args.storage_source == "hf" %} - - --model={{ args.model_id }} -{%- else %} - - --model=/mnt/models - - --served-model-name={{ args.model_id }} -{%- endif %} - - --port=8080 -{%- endif %} -{%- for key, val in (args.engine_args or {}).items() %} - {%- if val is boolean %} - {%- if val %} - - --{{ key }} - {%- endif %} - {%- else %} - - --{{ key }}={{ val }} - {%- endif %} -{%- endfor %} - env: - - name: USER - value: "{{ args.engine }}" - - name: XDG_CACHE_HOME - value: /tmp/.cache - - name: XDG_CONFIG_HOME - value: /tmp/.config -{%- if ctx.gpu_count > 1 %} - - name: NCCL_DEBUG - value: "WARN" -{%- endif %} -{%- if args.storage_source == "hf" %} - - name: HF_HUB_OFFLINE - value: "0" - - name: HOME - value: /mnt/models - - name: HF_HOME - value: /mnt/models -{%- if args.engine != "sglang" %} - - name: VLLM_CACHE_DIR - value: /mnt/models/.cache/vllm -{%- endif %} - - name: HF_DATASETS_CACHE - value: /mnt/models/.cache/huggingface/datasets - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: storage-config - key: HF_TOKEN -{%- else %} - - name: HOME - value: /tmp -{%- endif %} -{%- for env_item in ctx.env_vars_list %} - - name: {{ env_item.name }} - value: "{{ env_item.value }}" -{%- endfor %} - ports: - - containerPort: {{ args.engine_port }} - protocol: TCP -{%- if ctx.gpu_count > 1 %} - volumeMounts: - - name: shared-memory - mountPath: /dev/shm - volumes: - - name: shared-memory - emptyDir: - medium: Memory - sizeLimit: 64Gi -{%- endif %} -{%- endif %} From 70a55ead57b0d4a27cc0f623184a1bbb702111c9 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 16:13:41 -0400 Subject: [PATCH 0222/1072] fix: write hierarchical KPIs to separate file to preserve JSONL The hierarchical format transformation was overwriting kpis.json (JSONL) with a single indented JSON object, breaking the downstream csv-export step which expects JSON Lines input. Co-authored-by: Cursor --- projects/caliper/orchestration/postprocess.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 898a9929c..e94868754 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -542,17 +542,18 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Transform to hierarchical format + # Transform to hierarchical format and write to a separate file + # so the original JSONL is preserved for csv-export hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) - # Write back as JSON (schema v2) import json - with open(output_file, "w") as f: + hierarchical_file = output_file.with_suffix(".hierarchical.json") + with open(hierarchical_file, "w") as f: json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {hierarchical_file}" ) else: logger.warning("No KPI records found in output file") From 0f061ab4f73b65a259fcb3afe1d535ad56fca543 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 29 Jul 2026 23:35:22 -0400 Subject: [PATCH 0223/1072] add default trtllm_config structure to base config The config override system cannot create multiple levels of nested keys at once. Pre-populate trtllm_config with sensible defaults so FournosJob overrides for kv_cache_config, cuda_graph_config, and moe_config can resolve correctly. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 2b0d89124..c88c057f3 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -36,7 +36,19 @@ engines: nvidia: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc13 port: 8888 args: {} - trtllm_config: {} + trtllm_config: + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.90 + cuda_graph_config: + enable_padding: false + max_batch_size: 256 + enable_attention_dp: false + moe_config: + backend: CUTLASS + stream_interval: 0 + print_iter_log: false deploy: replicas: 1 From b69e3eb6de48c2f148f62496ce4f5b964332ca11 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 11:35:36 +0300 Subject: [PATCH 0224/1072] Add artifact export and MLflow config to nightly pipeline The nightly pipeline lacked an export-artifacts step, so logs never reached MLflow and GitHub notifications relied solely on submit.py. - Add export-artifacts finally step to pipeline-nightly.yaml - Write __test_labels__.yaml with outcome (launched/noop) in handler - Set nightly-specific MLflow experiment and run_naming overrides via FournosJob configOverrides in pr_args.py Co-authored-by: Cursor --- .../base/workflows/pipeline-nightly.yaml | 20 +++++++++++++++++++ projects/core/nightly/handler.py | 19 ++++++++++++++++++ .../mcp_gateway/orchestration/config.yaml | 5 +++++ projects/mcp_gateway/orchestration/pr_args.py | 9 +++++++++ 4 files changed, 53 insertions(+) diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml index a3e7a38ab..f39b7de29 100644 --- a/fournos/gitops/base/workflows/pipeline-nightly.yaml +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -41,3 +41,23 @@ spec: value: nightly - name: job-step-name value: 00__nightly + + finally: + - name: 01-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 01__export-artifacts diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 02a536efe..1d7112c07 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -13,6 +13,8 @@ import logging import os +import yaml + from projects.core.library import ci as ci_lib from projects.core.library import config, env from projects.core.library.config import requires @@ -116,6 +118,21 @@ def _create_fournos_job(project: str, version: str, source_cfg: dict, _cfg) -> N ) +def _write_test_labels() -> None: + """Write ``__test_labels__.yaml`` so the caliper export ``run_naming`` + templates can resolve the ``{outcome}`` placeholder.""" + result_file = env.ARTIFACT_DIR / "result.txt" + if result_file.exists(): + outcome = "noop" if result_file.read_text().strip() == "NO-OP" else "launched" + else: + outcome = "error" + + labels = {"outcome": outcome} + labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + labels_path.write_text(yaml.dump({"labels": labels})) + logger.info("Wrote test labels: %s", labels) + + def run(): """Entrypoint for the nightly phase.""" logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -137,3 +154,5 @@ def run(): logger.error("confirm-nightly failed: %s", e) ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") raise + finally: + _write_test_labels() diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 7b33d74c9..3357725c3 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,4 +1,9 @@ nightly: + caliper_export: + experiment: nightly-pipeline-experiment + run_naming: + prefix: "mcp-gw-nightly" + single_run: "{prefix}-{timestamp}-{outcome}" sources: ghcr: receiver: "projects.mcp_gateway.nightly_resolvers.ghcr_receiver.GHCRReceiver" diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 1a6f7d879..1523160bd 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -88,6 +88,15 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st source = parts[2] config_overrides["nightly.source"] = source config_overrides["fournos.job.pipeline_name"] = "nightly" + config_overrides["caliper.export.backend.mlflow.config.experiment"] = ( + "nightly-pipeline-experiment" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.prefix"] = ( + "mcp-gw-nightly" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.single_run"] = ( + "{prefix}-{timestamp}-{outcome}" + ) logger.info("Parsed nightly source from pipeline directive: %s", source) return config_overrides, parsed_directives From 1f64f22dcff33c9e27b42970bb6ecbac9e18ba5c Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 12:00:44 +0300 Subject: [PATCH 0225/1072] Decouple verifier experiment from export experiment The MLflow verifier now reads its experiment from nightly.mlflow_verifier.experiment (mcp-gw-cpt-ghcr) instead of caliper.export.backend.mlflow.config.experiment, so the export override to nightly-pipeline-experiment no longer affects version checks. Co-authored-by: Cursor --- projects/caliper/orchestration/mlflow_verifier.py | 4 ++-- projects/mcp_gateway/orchestration/config.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py index 7433c9e0b..bfcdb56e0 100644 --- a/projects/caliper/orchestration/mlflow_verifier.py +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -39,7 +39,7 @@ class CaliperMLflowVerifier(NightlyVerifier): """Check MLflow experiment runs for the last tested version. Config requirements (via @requires on __init__): - caliper.export.backend.mlflow.config.experiment + nightly.mlflow_verifier.experiment (experiment to query for previous runs) caliper.export.backend.mlflow.config.workspace caliper.export.backend.mlflow.secrets.vault (dict with 'name' and 'mlflow_secret') nightly.mlflow_verifier.version_pattern (regex with one capture group) @@ -47,7 +47,7 @@ class CaliperMLflowVerifier(NightlyVerifier): """ @requires( - experiment="caliper.export.backend.mlflow.config.experiment", + experiment="nightly.mlflow_verifier.experiment", workspace="caliper.export.backend.mlflow.config.workspace", vault_cfg="caliper.export.backend.mlflow.secrets.vault", version_pattern="nightly.mlflow_verifier.version_pattern", diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 3357725c3..2c8b22a34 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -12,6 +12,7 @@ nightly: mlflow_experiment: "mcp-gw-cpt-ghcr" verifier: "projects.caliper.orchestration.mlflow_verifier.CaliperMLflowVerifier" mlflow_verifier: + experiment: "mcp-gw-cpt-ghcr" version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" strip_prefix: "sha-" cluster: "forge-smoke-testing" From f088034abada14ad9e6c40f6eefef964c1c65f2e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 30 Jul 2026 12:06:05 +0300 Subject: [PATCH 0226/1072] Handle verifier bootstrap: treat failures as no previous version When the MLflow verifier cannot extract a version from existing run names (e.g. different naming convention during bootstrap), catch the exception and proceed as if no previous version exists, launching a new test run instead of failing the pipeline. Co-authored-by: Cursor --- projects/core/nightly/handler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py index 1d7112c07..e8e614659 100644 --- a/projects/core/nightly/handler.py +++ b/projects/core/nightly/handler.py @@ -57,7 +57,15 @@ def _confirm_nightly(project: str, image_version: str, source_cfg: dict) -> None verifier = verifier_cls() logger.info("Using verifier: %s", type(verifier).__name__) - last_version = verifier.get_last_tested_version() + try: + last_version = verifier.get_last_tested_version() + except Exception as e: + logger.warning( + "Verifier failed (bootstrap scenario?): %s. " + "Treating as no previous version — will launch a new run.", + e, + ) + last_version = "" logger.info("Last tested version: '%s'", last_version or "") if image_version == last_version: From 42ad31acec2639e8081bdf93fde3ba2431540a27 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:19:06 +0200 Subject: [PATCH 0227/1072] [kserve] toolbox: deploy_llmisvc: correctly resolve the kserve address --- projects/kserve/toolbox/deploy_llmisvc/main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index cf064757c..4e3fc062d 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -35,7 +35,7 @@ def run( *, namespace: str, inference_service_manifest_path: str, - gateway_status_address_name: str = "gateway-external", + gateway_status_address_name: str | None = "gateway-external", dry_run: bool = False, ) -> str: """ @@ -387,16 +387,25 @@ def resolve_endpoint_task(args, ctx): ctx.endpoint_url = endpoint_url write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") return f"Endpoint resolved: {endpoint_url}" - return False # Retry + return False, "No endpoint URL available" def try_resolve_endpoint_url( - *, namespace: str, inference_service_name: str, gateway_status_address_name: str + *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None ) -> str | None: payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) for address in payload.get("status", {}).get("addresses", []): - if address.get("name") == gateway_status_address_name and address.get("url"): + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 + if gateway_status_address_name is None: + if address.get("url"): + url = address["url"] + # Append port 8000 when not using gateway if no port is already specified + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + return url + # Otherwise, match by name + elif address.get("name") == gateway_status_address_name and address.get("url"): return address["url"] return None From 74f94fd5f9fd45116d420aceeeffc864d7c167dc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 22:39:53 +0200 Subject: [PATCH 0228/1072] [kserve] toolbox: deploy_llmisvc: improve wait_old_pods_gone --- projects/kserve/toolbox/deploy_llmisvc/main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 4e3fc062d..266323704 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -119,10 +119,20 @@ def delete_existing_service(args, ctx): def wait_old_pods_gone(args, ctx): """Wait for old llm-d pods to disappear""" - pods = oc_get_json( - "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + result = oc( + "get", + "pods", + "-n", + args.namespace, + "-l", + ctx.selector, + "--ignore-not-found=true", + "--no-headers", + check=False, ) - if not pods or not pods.get("items"): + + # Check if output is empty (no pods found) + if not result.stdout.strip(): return f"Old pods gone for {ctx.inference_service_name}" return False # Retry From ce0d075eb3981fd63e9c95962e89fae1ae1335d2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:00:30 +0200 Subject: [PATCH 0229/1072] [kserve] toolbox: prepare_hf_model_cache: wait for the pod to start running in a dedicated task --- .../toolbox/prepare_hf_model_cache/main.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index a25544e87..d6d502d8e 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -286,6 +286,44 @@ def create_download_job(args, ctx): return f"Download job {cache_spec['download_job_name']} created" +@retry(attempts=24, delay=5, backoff=1.0) # 24 attempts * 5 seconds = 120 seconds (2 minutes) +@task +def wait_for_pods_running(args, ctx): + """Wait for the download job pods to start running within 2 minutes""" + + if getattr(ctx, "cache_ready", False): + return "Skipping pod wait - cache already ready" + + cache_spec = ctx.cache_spec + job_name = cache_spec["download_job_name"] + namespace = cache_spec["namespace"] + + # Get pod status using plain text output + result = oc( + "get", + "pod", + "--no-headers", + "-l", + f"job-name={job_name}", + "-n", + namespace, + "-o", + "custom-columns=NAME:.metadata.name,STATUS:.status.phase", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False # No pods yet, retry + + # Keep waiting only if "Pending" appears in the output + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + # All pods are past Pending state + return "All pods past Pending state" + + @retry(attempts=30, delay=30, backoff=1.0) @task def wait_for_download(args, ctx): From ca2bceca5ac69cfdb31e2f63054d3c6d588f13c6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:12 +0200 Subject: [PATCH 0230/1072] [kserve] toolbox: capture_llmisvc_state: capture more information --- .../toolbox/capture_llmisvc_state/main.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index 12efa1935..060963447 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -26,6 +26,7 @@ def setup_directories(args, context): """Create the artifacts directory""" shell.mkdir("artifacts") + shell.mkdir("artifacts/logs") return "Artifacts directory created" @@ -121,7 +122,7 @@ def capture_namespace_pods(args, context): """Capture all pods in the namespace with wide output""" shell.run( f"oc get pods -owide -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status.txt", check=False, ) return "Namespace pods status captured" @@ -132,7 +133,7 @@ def capture_namespace_services(args, context): """Capture all services in the namespace""" shell.run( f"oc get svc -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.services.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.services.status.txt", check=False, ) return "Namespace services captured" @@ -173,20 +174,19 @@ def capture_pod_logs(args, context): if not pod_names or not result.stdout.strip(): return "No pods found to capture logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod logs captured for {len(pod_names)} pods" + return f"Pod logs captured for {captured_count} pods in dedicated files" @task @@ -195,26 +195,26 @@ def capture_pod_previous_logs(args, context): result = shell.run( f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].metadata.name}}"', check=False, + log_stdout=False, ) pod_names = result.stdout.strip().split() if not pod_names or not result.stdout.strip(): return "No pods found to capture previous logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.previous.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Previous logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.previous.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod previous logs captured for {len(pod_names)} pods" + return f"Pod previous logs captured for {captured_count} pods in dedicated files" @task From 049bdcc16f358d141d04daa8971cc34a8cf37f15 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:26 +0200 Subject: [PATCH 0231/1072] [kserve] toolbox: deploy_llmisvc: capture more information --- .../kserve/toolbox/deploy_llmisvc/main.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 266323704..d2768aa9e 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -267,6 +267,197 @@ def capture_replicaset_description(args, ctx): return f"Failed to capture ReplicaSet description: {e}" +@always +@task +def capture_final_llmisvc_yaml(args, ctx): + """Capture the final YAML state of the LLMInferenceService""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" + + # Capture final YAML state + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + llmisvc_yaml_path = artifacts_dir / "llmisvc_final.yaml" + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured final LLMInferenceService YAML to {llmisvc_yaml_path}" + + except Exception as e: + return f"Failed to capture final LLMInferenceService YAML: {e}" + + +@always +@task +def capture_pod_status(args, ctx): + """Capture pod status for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture pod status with wide output + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + log_stdout=False, + check=False, + ) + + pod_status_path = artifacts_dir / "pod_status.txt" + with open(pod_status_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod status to {pod_status_path}" + + except Exception as e: + return f"Failed to capture pod status: {e}" + + +@always +@task +def capture_pod_descriptions(args, ctx): + """Capture pod descriptions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Get pod names + pod_result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "jsonpath={.items[*].metadata.name}", + log_stdout=False, + check=False, + ) + + pod_names = pod_result.stdout.strip().split() + if not pod_names or not pod_result.stdout.strip(): + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("No pods found for the service") + return f"No pods found, wrote empty file to {pod_desc_path}" + + # Describe each pod + pod_descriptions = [] + for pod_name in pod_names: + describe_result = oc( + "describe", + "pod", + pod_name, + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + pod_descriptions.append( + f"=== Description for pod: {pod_name} ===\n{describe_result.stdout}" + ) + + # Save all pod descriptions + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("\n\n".join(pod_descriptions)) + + return f"Captured descriptions for {len(pod_names)} pods to {pod_desc_path}" + + except Exception as e: + return f"Failed to capture pod descriptions: {e}" + + +@always +@task +def capture_pod_yaml(args, ctx): + """Capture pod YAML definitions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture all pod YAMLs + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + pod_yaml_path = artifacts_dir / "pod_definitions.yaml" + with open(pod_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod YAML definitions to {pod_yaml_path}" + + except Exception as e: + return f"Failed to capture pod YAML: {e}" + + @task def query_service_status(args, ctx): """Query the status of the LLMInferenceService""" From 35efb5176209ee3db411c55e1e4b14184de7dc86 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:37:10 +0200 Subject: [PATCH 0232/1072] [kserve] toolbox: prepare_hf_model_cache: wait 1h for the download completion --- projects/kserve/toolbox/prepare_hf_model_cache/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index d6d502d8e..1e01c1737 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -324,7 +324,7 @@ def wait_for_pods_running(args, ctx): return "All pods past Pending state" -@retry(attempts=30, delay=30, backoff=1.0) +@retry(attempts=120, delay=30, backoff=1.0) @task def wait_for_download(args, ctx): """Wait for the download job to complete""" From d8f7538b888f0d1f7f296c119009ff0833ab3985 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 14:48:32 +0200 Subject: [PATCH 0233/1072] [kserve] toolbox: deploy_llmisvc: add flag to wait_pods_scheduled --- .../kserve/toolbox/deploy_llmisvc/main.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index d2768aa9e..8d75691e2 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -37,6 +37,7 @@ def run( inference_service_manifest_path: str, gateway_status_address_name: str | None = "gateway-external", dry_run: bool = False, + wait_pods_scheduled: bool = False, ) -> str: """ Deploy an LLMInferenceService and wait for its endpoint. @@ -46,6 +47,7 @@ def run( inference_service_manifest_path: Path to the InferenceService YAML manifest file gateway_status_address_name: Gateway status address name for endpoint resolution dry_run: If True, only prepare the manifest without deploying + wait_pods_scheduled: If True, wait for all pods to be scheduled before checking service readiness """ ctx = execute_tasks(locals()) @@ -521,6 +523,40 @@ def query_service_message(args, ctx): return "No Ready condition found in status" +@retry(attempts=999999, delay=30, backoff=1.0) +@task +def wait_pods_scheduled(args, ctx): + """Wait for all pods to be scheduled (optional task)""" + + # Check if this task is enabled + if not args.wait_pods_scheduled: + return "Pod scheduling wait disabled by parameter" + + service_name = ctx.inference_service_name + + # Get pod status using plain text output + result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "--no-headers", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False, "No pods found for the service yet" + + # Keep waiting if any pod is Pending + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + return f"All pods for {service_name} are scheduled successfully" + + @retry(attempts=90, delay=10, backoff=1.0) @task def wait_service_ready(args, ctx): From b95774139a18ceab77f18f4f79513ce915239a81 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:03:26 +0200 Subject: [PATCH 0234/1072] [kserve] toolbox: deploy_llmisvc: capture_final_llmisvc_yaml: better error handling --- .../kserve/toolbox/deploy_llmisvc/main.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 8d75691e2..3f5433410 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path import yaml @@ -24,6 +25,8 @@ from .on_failure_helpers import on_wait_pods_appear_failure +logger = logging.getLogger(__name__) + def load_yaml(path: Path): with path.open(encoding="utf-8") as handle: @@ -301,6 +304,30 @@ def capture_final_llmisvc_yaml(args, ctx): ) llmisvc_yaml_path = artifacts_dir / "llmisvc_final.yaml" + + if result.returncode != 0: + # Handle the case where the LLMInferenceService is not found + if "not found" in result.stderr.lower(): + logger.warning( + f"LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write( + f"# LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'\n" + ) + f.write(f"# Error: {result.stderr.strip()}\n") + return f"LLMInferenceService not found, logged error to {llmisvc_yaml_path}" + else: + # Handle other error cases + logger.error( + f"Failed to get LLMInferenceService '{service_name}': {result.stderr.strip()}" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(f"# Error getting LLMInferenceService '{service_name}'\n") + f.write(f"# Error: {result.stderr.strip()}\n") + return f"Error getting LLMInferenceService, logged error to {llmisvc_yaml_path}" + + # Success case - write the YAML content with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: f.write(result.stdout) From e9c25ddf320cd65a0578c08d6b137a38a440f995 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 12:04:22 +0200 Subject: [PATCH 0235/1072] [kserve] toolbox: deploy_llmisvc: abort on Pod restart --- .../kserve/toolbox/deploy_llmisvc/main.py | 400 ++++++++++-------- 1 file changed, 213 insertions(+), 187 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 3f5433410..5dd994582 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -169,6 +169,219 @@ def wait_pods_appear(args, ctx): return False # Retry +@task +def query_service_status(args, ctx): + """Query the status of the LLMInferenceService""" + + service_name = ctx.inference_service_name + + # Query only the Ready condition status + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')].status}", + log_stdout=False, + ) + + ready_status = result.stdout.strip() + ctx.is_ready = ready_status == "True" + + if ctx.is_ready: + return f"LLMInferenceService {service_name} status: Ready" + else: + return f"LLMInferenceService {service_name} status: Not Ready" + + +@task +def query_service_message(args, ctx): + """Query detailed message from LLMInferenceService""" + + service_name = ctx.inference_service_name + + # Query the Ready condition details + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')]}", + log_stdout=False, + ) + + if result.stdout.strip(): + try: + import json + + condition = json.loads(result.stdout) + reason = condition.get("reason", "Unknown") + message = condition.get("message", "No message") + + if not ctx.is_ready: + return f"Not ready - Reason: {reason}, Message: {message}" + else: + return "Ready - Service is operational" + except (json.JSONDecodeError, KeyError) as e: + return f"Failed to parse Ready condition: {e}" + else: + return "No Ready condition found in status" + + +@retry(attempts=999999, delay=30, backoff=1.0) +@task +def wait_pods_scheduled(args, ctx): + """Wait for all pods to be scheduled (optional task)""" + + # Check if this task is enabled + if not args.wait_pods_scheduled: + return "Pod scheduling wait disabled by parameter" + + service_name = ctx.inference_service_name + + # Get pod status using plain text output + result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "--no-headers", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False, "No pods found for the service yet" + + # Keep waiting if any pod is Pending + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + return f"All pods for {service_name} are scheduled successfully" + + +@retry(attempts=90, delay=10, backoff=1.0) +@task +def wait_service_ready(args, ctx): + """Wait for LLMInferenceService to be ready""" + + service_name = ctx.inference_service_name + + # Query the current status and show diagnostic info + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Ready')]}", + ) + + # Also show pod status for debugging + oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + ) + + # Check for pod restarts and abort if any pods have restarted + restart_result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "-o", + "jsonpath={range .items[*]}{.metadata.name}:{.status.containerStatuses[*].restartCount}{'\\n'}{end}", + ) + + if restart_result.stdout.strip(): + for line in restart_result.stdout.strip().split("\n"): + if ":" in line: + pod_name, restart_counts = line.split(":", 1) + # Check if any container has restarted + counts = restart_counts.split() + for count_str in counts: + try: + if int(count_str) > 0: + raise RuntimeError( + f"Pod {pod_name} has restarted (restart count: {count_str}). Aborting wait due to pod restart." + ) + except ValueError: + # Skip non-numeric restart counts + pass + + if result.stdout.strip(): + try: + import json + + condition = json.loads(result.stdout) + status = condition.get("status", "Unknown") + reason = condition.get("reason", "Unknown") + message = condition.get("message", "No message") + + if status == "True": + return f"LLMInferenceService {service_name} is ready" + else: + return ( + False, + f"Service not ready - Status: {status}, Reason: {reason}, Message: {message}", + ) + + except (json.JSONDecodeError, KeyError) as e: + return (False, f"Failed to parse Ready condition: {e}") + else: + return (False, f"No Ready condition found in status for {service_name}") + + +def try_resolve_endpoint_url( + *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None +) -> str | None: + payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) + + for address in payload.get("status", {}).get("addresses", []): + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 + if gateway_status_address_name is None: + if address.get("url"): + url = address["url"] + # Append port 8000 when not using gateway if no port is already specified + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + return url + # Otherwise, match by name + elif address.get("name") == gateway_status_address_name and address.get("url"): + return address["url"] + return None + + +@retry(attempts=90, delay=10, backoff=1.0) +@task +def resolve_endpoint_task(args, ctx): + """Resolve the gateway endpoint URL""" + + endpoint_url = try_resolve_endpoint_url( + namespace=args.namespace, + inference_service_name=ctx.inference_service_name, + gateway_status_address_name=args.gateway_status_address_name, + ) + if endpoint_url: + ctx.endpoint_url = endpoint_url + write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") + return f"Endpoint resolved: {endpoint_url}" + return False, "No endpoint URL available" + + @always @task def capture_llmisv_description(args, ctx): @@ -487,192 +700,5 @@ def capture_pod_yaml(args, ctx): return f"Failed to capture pod YAML: {e}" -@task -def query_service_status(args, ctx): - """Query the status of the LLMInferenceService""" - - service_name = ctx.inference_service_name - - # Query only the Ready condition status - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')].status}", - log_stdout=False, - ) - - ready_status = result.stdout.strip() - ctx.is_ready = ready_status == "True" - - if ctx.is_ready: - return f"LLMInferenceService {service_name} status: Ready" - else: - return f"LLMInferenceService {service_name} status: Not Ready" - - -@task -def query_service_message(args, ctx): - """Query detailed message from LLMInferenceService""" - - service_name = ctx.inference_service_name - - # Query the Ready condition details - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')]}", - log_stdout=False, - ) - - if result.stdout.strip(): - try: - import json - - condition = json.loads(result.stdout) - reason = condition.get("reason", "Unknown") - message = condition.get("message", "No message") - - if not ctx.is_ready: - return f"Not ready - Reason: {reason}, Message: {message}" - else: - return "Ready - Service is operational" - except (json.JSONDecodeError, KeyError) as e: - return f"Failed to parse Ready condition: {e}" - else: - return "No Ready condition found in status" - - -@retry(attempts=999999, delay=30, backoff=1.0) -@task -def wait_pods_scheduled(args, ctx): - """Wait for all pods to be scheduled (optional task)""" - - # Check if this task is enabled - if not args.wait_pods_scheduled: - return "Pod scheduling wait disabled by parameter" - - service_name = ctx.inference_service_name - - # Get pod status using plain text output - result = oc( - "get", - "pods", - "-l", - ctx.selector, - "-n", - args.namespace, - "--no-headers", - check=False, - log_stdout=False, - ) - - if not result.stdout.strip(): - return False, "No pods found for the service yet" - - # Keep waiting if any pod is Pending - if "Pending" in result.stdout: - return False, "Waiting for pods to exit Pending state" - - return f"All pods for {service_name} are scheduled successfully" - - -@retry(attempts=90, delay=10, backoff=1.0) -@task -def wait_service_ready(args, ctx): - """Wait for LLMInferenceService to be ready""" - - service_name = ctx.inference_service_name - - # Query the current status and show diagnostic info - result = oc( - "get", - "llminferenceservice", - service_name, - "-n", - args.namespace, - "-o", - "jsonpath={.status.conditions[?(@.type=='Ready')]}", - log_stdout=True, - ) - - # Also show pod status for debugging - oc( - "get", - "pods", - "-l", - ctx.selector, - "-n", - args.namespace, - log_stdout=True, # Show pod status in logs - ) - - if result.stdout.strip(): - try: - import json - - condition = json.loads(result.stdout) - status = condition.get("status", "Unknown") - reason = condition.get("reason", "Unknown") - message = condition.get("message", "No message") - - if status == "True": - return f"LLMInferenceService {service_name} is ready" - else: - return ( - False, - f"Service not ready - Status: {status}, Reason: {reason}, Message: {message}", - ) - - except (json.JSONDecodeError, KeyError) as e: - return (False, f"Failed to parse Ready condition: {e}") - else: - return (False, f"No Ready condition found in status for {service_name}") - - -@retry(attempts=90, delay=10, backoff=1.0) -@task -def resolve_endpoint_task(args, ctx): - """Resolve the gateway endpoint URL""" - - endpoint_url = try_resolve_endpoint_url( - namespace=args.namespace, - inference_service_name=ctx.inference_service_name, - gateway_status_address_name=args.gateway_status_address_name, - ) - if endpoint_url: - ctx.endpoint_url = endpoint_url - write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") - return f"Endpoint resolved: {endpoint_url}" - return False, "No endpoint URL available" - - -def try_resolve_endpoint_url( - *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None -) -> str | None: - payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) - - for address in payload.get("status", {}).get("addresses", []): - # When gateway_status_address_name is None, return the first address with a URL and append port 8000 - if gateway_status_address_name is None: - if address.get("url"): - url = address["url"] - # Append port 8000 when not using gateway if no port is already specified - if ":" not in url.split("/")[-1]: # Check if no port in the hostname part - url = f"{url}:8000" - return url - # Otherwise, match by name - elif address.get("name") == gateway_status_address_name and address.get("url"): - return address["url"] - return None - - if __name__ == "__main__": run.main() From 3d8055036af27cad11c0eb775d4373d972a0fb68 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:48:28 +0200 Subject: [PATCH 0236/1072] [kserve] toolbox: deploy_llmisvc: improve the URL resolution --- .../kserve/toolbox/deploy_llmisvc/main.py | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 5dd994582..553582617 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -348,24 +348,112 @@ def wait_service_ready(args, ctx): def try_resolve_endpoint_url( *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None ) -> str | None: + logger.info( + f"=== Resolving endpoint URL for {inference_service_name} in namespace {namespace} ===" + ) + logger.info(f"Target gateway_status_address_name: {gateway_status_address_name}") + payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) - for address in payload.get("status", {}).get("addresses", []): + # Log the entire status section for debugging + status = payload.get("status", {}) + logger.info(f"Status section keys found: {list(status.keys())}") + + # Check status.address first + status_address = status.get("address") + logger.info(f"status.address content: {status_address}") + + if status_address: + logger.info("✓ status.address exists") + if isinstance(status_address, dict): + logger.info("✓ status.address is a dict") + if status_address.get("url"): + url = status_address["url"] + logger.info(f"✓ Found URL in status.address: '{url}'") + + # When gateway_status_address_name is None, append port 8000 if needed + if gateway_status_address_name is None: + logger.info("Mode: No gateway (will append port 8000 if needed)") + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + logger.info(f"✓ Appended port 8000: '{url}'") + else: + logger.info("✓ URL already has port, using as-is") + else: + logger.info( + f"Mode: Gateway '{gateway_status_address_name}' (no port modification)" + ) + + logger.info(f"🎯 RESOLVED from status.address: '{url}'") + return url + else: + logger.info("✗ status.address has no 'url' field") + logger.info(f" Available fields: {list(status_address.keys())}") + else: + logger.info(f"✗ status.address is not a dict, type: {type(status_address)}") + else: + logger.info("✗ No status.address found") + + # Fallback to existing status.addresses logic + logger.info("--- Falling back to status.addresses lookup ---") + status_addresses = status.get("addresses", []) + logger.info(f"status.addresses content: {status_addresses}") + + if not status_addresses: + logger.info("✗ No status.addresses found - RESOLUTION FAILED") + return None + + logger.info(f"✓ Found {len(status_addresses)} address(es) in status.addresses") + + for i, address in enumerate(status_addresses): + logger.info(f"Checking address[{i}]: {address}") + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 if gateway_status_address_name is None: + logger.info(" Mode: No gateway filter (looking for any URL)") if address.get("url"): url = address["url"] + logger.info(f" ✓ Found URL in address[{i}]: '{url}'") # Append port 8000 when not using gateway if no port is already specified if ":" not in url.split("/")[-1]: # Check if no port in the hostname part url = f"{url}:8000" + logger.info(f" ✓ Appended port 8000: '{url}'") + else: + logger.info(" ✓ URL already has port, using as-is") + + logger.info(f"🎯 RESOLVED from status.addresses[{i}]: '{url}'") return url + else: + logger.info(f" ✗ Address[{i}] has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") # Otherwise, match by name - elif address.get("name") == gateway_status_address_name and address.get("url"): - return address["url"] + else: + address_name = address.get("name") + logger.info( + f" Mode: Gateway filter (looking for name='{gateway_status_address_name}')" + ) + logger.info(f" Address[{i}] name: '{address_name}'") + + if address_name == gateway_status_address_name: + logger.info(f" ✓ Name matches '{gateway_status_address_name}'") + if address.get("url"): + url = address["url"] + logger.info(f" ✓ Found URL: '{url}'") + logger.info(f"🎯 RESOLVED from status.addresses[{i}] by name: '{url}'") + return url + else: + logger.info(" ✗ Matching address has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") + else: + logger.info( + f" ✗ Name mismatch: '{address_name}' != '{gateway_status_address_name}'" + ) + + logger.info("❌ RESOLUTION FAILED - No suitable URL found in any address") return None -@retry(attempts=90, delay=10, backoff=1.0) +@retry(attempts=30, delay=10, backoff=1.0) @task def resolve_endpoint_task(args, ctx): """Resolve the gateway endpoint URL""" From 02247d69a6486928ddcbb4ec9be22c7ad9e8a72a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:20:56 +0200 Subject: [PATCH 0237/1072] [kserve] toolbox: deploy_llmisvc: also wait on SchedulingGated --- projects/kserve/toolbox/deploy_llmisvc/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 553582617..59ba34f83 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -259,10 +259,13 @@ def wait_pods_scheduled(args, ctx): if not result.stdout.strip(): return False, "No pods found for the service yet" - # Keep waiting if any pod is Pending + # Keep waiting if any pod is Pending or SchedulingGated if "Pending" in result.stdout: return False, "Waiting for pods to exit Pending state" + if "SchedulingGated" in result.stdout: + return False, "Waiting for pods to exit SchedulingGated state" + return f"All pods for {service_name} are scheduled successfully" @@ -304,6 +307,7 @@ def wait_service_ready(args, ctx): args.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}:{.status.containerStatuses[*].restartCount}{'\\n'}{end}", + log_stdout=False, ) if restart_result.stdout.strip(): From ef3abf5ec53094582fe3a00a06421860e314e517 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:58:14 +0200 Subject: [PATCH 0238/1072] [kserve] toolbox: deploy_llmisvc: expose the try_resolve_endpoint_url method --- projects/kserve/toolbox/deploy_llmisvc/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/__init__.py b/projects/kserve/toolbox/deploy_llmisvc/__init__.py index e69de29bb..68f67b1fb 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/__init__.py +++ b/projects/kserve/toolbox/deploy_llmisvc/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from projects.kserve.toolbox.deploy_llmisvc.main import try_resolve_endpoint_url + +__all__ = ["try_resolve_endpoint_url"] From 8ef80c1cca50415add3ae364b681419202351d56 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 21:04:34 +0200 Subject: [PATCH 0239/1072] [kserve] toolbox: deploy_llmisvc: save the deploy/rs/pod description --- .../kserve/toolbox/deploy_llmisvc/main.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 59ba34f83..c3efade89 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -642,6 +642,42 @@ def capture_final_llmisvc_yaml(args, ctx): return f"Failed to capture final LLMInferenceService YAML: {e}" +@always +@task +def capture_workload_overview(args, ctx): + """Capture deployment, replicaset, and pod overview for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + workload_overview_path = artifacts_dir / "workload_overview.txt" + + # Capture deployment, replicaset, and pod overview + oc( + "get", + "deploy,rs,pod", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + check=False, + stdout_dest=workload_overview_path, + ) + + return f"Captured workload overview to {workload_overview_path}" + + @always @task def capture_pod_status(args, ctx): From 96b00a0a20a109893cc133a57101d731c40568ff Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 17:28:11 +0200 Subject: [PATCH 0240/1072] [guidellm] toolbox: run_smoke_request: save the pod info in a file --- projects/guidellm/toolbox/run_smoke_request/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index 5905beff2..e131e995c 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -247,6 +247,7 @@ def capture_smoke_pod_debug_info(args, ctx): "-o", "yaml", check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.yaml", ) # Capture pod description From 68b26a81a640bb6159d41fe4f007b935bdcd608c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:08:43 +0200 Subject: [PATCH 0241/1072] [guidellm] toolbox: run_smoke_request: don't wait for the pod deletion --- projects/guidellm/toolbox/run_smoke_request/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index e131e995c..eba195b84 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -284,6 +284,7 @@ def cleanup_smoke_pod(args, ctx): "-n", args.namespace, "--ignore-not-found=true", + "--wait=false", check=False, ) return f"Cleaned up smoke pod {ctx.pod_name}" From bcde3d06b9af5efc72a6980208c6f5ede37155e4 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:18:36 +0200 Subject: [PATCH 0242/1072] [guidellm] toolbox: run_guidellm_benchmark: give more time --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index fa82ff1ec..12b6f3446 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -147,7 +147,7 @@ def create_guidellm_resources_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} created" -@retry(attempts=180, delay=10, backoff=1.0) +@retry(attempts=1080, delay=10, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" From b4e8c2cf5666b55ad9902c508425ef4dec4c0abe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:04:58 +0200 Subject: [PATCH 0243/1072] [guidellm] toolbox: run_guidellm_benchmark: set the owner of the PVC --- .../toolbox/run_guidellm_benchmark/main.py | 38 ++++++++++++++----- .../toolbox/run_guidellm_benchmark/utils.py | 13 ++++++- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 12b6f3446..c7fe0b2ae 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -120,19 +120,12 @@ def _best_effort_delete(description: str, *oc_args: str) -> None: @task def create_guidellm_resources_task(args, ctx): - """Create the GuideLLM benchmark PVC and job""" + """Create the GuideLLM benchmark job and PVC with job as owner""" # Ensure src directory exists (args.artifact_dir / "src").mkdir(parents=True, exist_ok=True) - oc_apply( - args.artifact_dir / "src" / "guidellm-pvc.yaml", - render_guidellm_pvc_from_parts( - namespace=ctx.target_namespace, - name=ctx.benchmark_name, - pvc_size=args.pvc_size, - ), - ) + # Create the job first oc_apply( args.artifact_dir / "src" / "guidellm-job.yaml", render_guidellm_job_from_parts( @@ -144,7 +137,32 @@ def create_guidellm_resources_task(args, ctx): hf_token_secret=args.hf_token_secret, ), ) - return f"GuideLLM benchmark {ctx.benchmark_name} created" + + # Get the job metadata for owner reference + job_data = oc_get_json("job", name=ctx.benchmark_name, namespace=ctx.target_namespace) + + # Create owner reference from job metadata + owner_reference = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": job_data["metadata"]["name"], + "uid": job_data["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } + + # Create the PVC with job as owner + oc_apply( + args.artifact_dir / "src" / "guidellm-pvc.yaml", + render_guidellm_pvc_from_parts( + namespace=ctx.target_namespace, + name=ctx.benchmark_name, + pvc_size=args.pvc_size, + owner_reference=owner_reference, + ), + ) + + return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" @retry(attempts=1080, delay=10, backoff=1.0) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 7cab6a2c2..6366acce8 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -146,13 +146,16 @@ def _build_multi_run_script(*, endpoint_url: str, runs: list[GuideLLMRun]) -> st return "\n".join(lines) -def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) -> dict[str, Any]: +def render_guidellm_pvc_from_parts( + *, namespace: str, name: str, pvc_size: str, owner_reference: dict[str, Any] | None = None +) -> dict[str, Any]: """Render a GuideLL-M PVC manifest from individual components. Args: namespace: Target namespace name: Name of the benchmark job and PVC pvc_size: Size of the PVC + owner_reference: Optional owner reference to set (e.g., for job ownership) Returns: PVC manifest as dict @@ -165,7 +168,13 @@ def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) "pvc_size": pvc_size, }, ) - return yaml.safe_load(rendered_yaml) + manifest = yaml.safe_load(rendered_yaml) + + # Add owner reference if provided + if owner_reference: + manifest["metadata"]["ownerReferences"] = [owner_reference] + + return manifest def render_guidellm_job_from_parts( From c5547eb47f07ad895b6e467d84f91e98ef10e8ae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 09:44:13 +0200 Subject: [PATCH 0244/1072] [caliper] engine: parameter_matrix: don't truncate long parameters that merges unrelated legend names ... --- projects/caliper/engine/parameter_matrix.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/caliper/engine/parameter_matrix.py b/projects/caliper/engine/parameter_matrix.py index 69c95d3ee..03e720e10 100644 --- a/projects/caliper/engine/parameter_matrix.py +++ b/projects/caliper/engine/parameter_matrix.py @@ -237,8 +237,4 @@ def create_legend_name( legend_name = ", ".join(param_pairs) - # Truncate if too long - if len(legend_name) > max_length: - legend_name = legend_name[: max_length - 3] + "..." - return legend_name From b7f46d8e8492e851e31e899ce954bfed6e1dc636 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 10:17:24 +0200 Subject: [PATCH 0245/1072] [llm_d] tests: test_deployment_profiles: add a 60s timeout --- .../llm_d/tests/test_deployment_profiles.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 631ab48a0..0663760a3 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -67,13 +67,25 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): ci_script = PROJECT_ROOT / "projects" / "llm_d" / "orchestration" / "ci.py" cmd = [str(ci_script), "--preset", preset, "test"] - result = subprocess.run( - cmd, - capture_output=True, - text=True, - cwd=PROJECT_ROOT, - env=env_vars, - ) + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + env=env_vars, + timeout=60, # 60-second timeout + ) + except subprocess.TimeoutExpired as e: + # Show stdout when timeout occurs + stdout_output = e.stdout.decode() if e.stdout else "No stdout available" + stderr_output = e.stderr.decode() if e.stderr else "No stderr available" + pytest.fail( + f"CI command timed out after 60 seconds for preset {preset}:\n" + f"Command: {' '.join(cmd)}\n\n" + f"STDOUT (before timeout):\n{stdout_output}\n\n" + f"STDERR (before timeout):\n{stderr_output}\n\n" + ) # Check that the command succeeded if result.returncode != 0: From b4577af93e4e4b43da8dab7fcfefb4d7fe564a26 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 09:46:07 +0200 Subject: [PATCH 0246/1072] [caliper] postprocess: helpers: visualization_utils: update the size/scale of the plots --- .../helpers/visualization_utils.py | 163 +++++------------- 1 file changed, 45 insertions(+), 118 deletions(-) diff --git a/projects/caliper/postprocess/helpers/visualization_utils.py b/projects/caliper/postprocess/helpers/visualization_utils.py index 68d45c361..a6e6f4c2b 100644 --- a/projects/caliper/postprocess/helpers/visualization_utils.py +++ b/projects/caliper/postprocess/helpers/visualization_utils.py @@ -3,8 +3,11 @@ from __future__ import annotations import base64 +import logging from pathlib import Path +logger = logging.getLogger(__name__) + def save_figure( fig, @@ -12,8 +15,8 @@ def save_figure( filename: str, as_image: bool = True, report_number: int | None = None, - width: int = 800, - height: int = 500, + width: int = 1200, + height: int = 650, ) -> str | None: """ Save a plotly figure as either an image or HTML file with optional report numbering. @@ -38,135 +41,34 @@ def save_figure( final_filename = filename if as_image: - print(f"💾 Saving {final_filename} as PNG image...") + logger.info(f"Saving {final_filename} as PNG image...") output_file = output_dir / f"{final_filename}.png" - fig.write_image(output_file, width=width, height=height, scale=2) + fig.write_image(output_file, width=width, height=height) else: - print(f"💾 Saving {final_filename} as full-page interactive HTML...") + logger.info(f"Saving {final_filename} as full-page interactive HTML...") output_file = output_dir / f"{final_filename}.html" - # Configure the figure for full-page display + # Configure figure for full-screen responsive behavior fig.update_layout( autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - ) - - # Save with full-page configuration - fig.write_html( - output_file, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, + width=None, # Remove any fixed width + height=None, # Remove any fixed height ) + fig.write_html(output_file) - # Make it full page - _make_html_full_page(output_file) - - print(f"✅ {final_filename} saved successfully") + logger.info(f"{final_filename} saved successfully") return str(output_file) except Exception as e: final_filename = ( f"report_{report_number:02d}_{filename}" if report_number is not None else filename ) - print(f"❌ Failed to save figure {final_filename}: {e}") + logger.error(f"Failed to save figure {final_filename}: {e}") return None -def _make_html_full_page(html_file_path: str) -> None: - """ - Modify an HTML file to make the plot full page by adding custom CSS. - - Args: - html_file_path: Path to the HTML file to modify - """ - try: - with open(html_file_path, encoding="utf-8") as f: - content = f.read() - - # Insert full-page CSS styles - full_page_css = """ - -""" - - # Insert the CSS before the closing tag - content = content.replace("", f"{full_page_css}") - - with open(html_file_path, "w", encoding="utf-8") as f: - f.write(content) - - except Exception as e: - print(f"⚠️ Warning: Failed to make HTML full page: {e}") - - -def write_full_page_html(fig, output_file_path: str, title: str = "Plot") -> bool: - """ - Save a Plotly figure as a full-page HTML file. - - Args: - fig: Plotly figure object - output_file_path: Path where to save the HTML file - title: Title for the HTML page - - Returns: - True if successful, False otherwise - """ - try: - # Configure the figure for full-page display - fig.update_layout( - autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - title_text=title if title != "Plot" else None, - ) - - # Save with full-page configuration - fig.write_html( - output_file_path, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, - ) - - # Make it full page - _make_html_full_page(output_file_path) - return True - - except Exception as e: - print(f"❌ Failed to save full-page HTML: {e}") - return False - - def figure_to_base64( - fig, width: int = 800, height: int = 500, plot_name: str = "plot" + fig, width: int = 1200, height: int = 650, plot_name: str = "plot" ) -> str | None: """ Convert a plotly figure to base64-encoded PNG for embedding in HTML. @@ -181,20 +83,20 @@ def figure_to_base64( Base64-encoded image string with data URI prefix, or None if failed """ try: - print(f"🖼️ Converting {plot_name} to high-quality PNG ({width}x{height})...") + logger.info(f"Converting {plot_name} to high-quality PNG ({width}x{height})...") # Convert figure to PNG bytes - img_bytes = fig.to_image(format="png", width=width, height=height, scale=2) + img_bytes = fig.to_image(format="png", width=width, height=height) - print(f"📦 Encoding {plot_name} as base64 for HTML embedding...") + logger.info(f"Encoding {plot_name} as base64 for HTML embedding...") # Encode as base64 img_base64 = base64.b64encode(img_bytes).decode() - print(f"✅ {plot_name} image ready ({len(img_base64) // 1024}KB)") + logger.info(f"{plot_name} image ready ({len(img_base64) // 1024}KB)") return f"data:image/png;base64,{img_base64}" except Exception as e: - print(f"❌ Failed to convert {plot_name} to base64: {e}") + logger.error(f"Failed to convert {plot_name} to base64: {e}") return None @@ -265,3 +167,28 @@ def create_report_title_display(base_title: str, report_number: int | None = Non return f"Report {report_number:02d}: {base_title}" else: return base_title + + +def write_full_page_html(fig, output_path: str, title: str) -> bool: + """ + Write a plotly figure as a full-page HTML file. + + Args: + fig: Plotly figure object + output_path: Full path to the output HTML file + title: Title for the HTML document (currently unused but kept for compatibility) + + Returns: + True if successful, False otherwise + """ + try: + output_file = Path(output_path) + output_dir = output_file.parent + filename = output_file.stem # Get filename without extension + + # Use save_figure to write HTML + result = save_figure(fig, output_dir, filename, as_image=False) + return result is not None + except Exception as e: + logger.error(f"Failed to write full-page HTML to {output_path}: {e}") + return False From 5ebeae81640a96e36726009e4b744c77e760864a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:42:54 +0200 Subject: [PATCH 0247/1072] [caliper] cli: commands: don't swallow the stack trace --- projects/caliper/cli/commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 3bcafe341..181d8c03b 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -173,13 +173,19 @@ def parse_cmd( click.echo("⚠️ No test directories found") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() status.update( { "success": False, "error": str(e), + "traceback": full_traceback, } ) click.echo(f"parse failed: {e}", err=True) + click.echo("Full traceback:", err=True) + click.echo(full_traceback, err=True) finally: # Write status file if requested if status_file: From b80af82167e94a6bf91ec3fb6a2dd2e1c2487bdf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:43:13 +0200 Subject: [PATCH 0248/1072] [caliper] engine: parse: show the parsing duration --- projects/caliper/engine/parse.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index 132e4428e..3a81f8fd0 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import time from pathlib import Path from projects.caliper.engine.cache import ( @@ -48,9 +49,15 @@ def run_parse( cache_refs = [] all_warnings = [] + # Track timing statistics + parse_start_time = time.time() + cached_count = 0 + parsed_count = 0 + parse_fn = plugin.parse for node in nodes: + start_time = time.time() test_base_dir = node.directory cache_file = cache_path_for_test_base(test_base_dir, plugin_module) fp = fingerprint_test_base(test_base_dir, plugin_module) @@ -71,6 +78,10 @@ def run_parse( # Use cached records all_records.extend(cached_records) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + cached_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed from cache in {elapsed_time:.3f}s: {relative_path}") else: # Parse this test base result = parse_fn([node]) # Parse just this node @@ -88,6 +99,16 @@ def run_parse( fingerprint=fp, ) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + parsed_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed fresh in {elapsed_time:.3f}s: {relative_path}") + + # Log timing summary + total_parse_time = time.time() - parse_start_time + logger.info( + f"⏱️ Parse timing summary: {total_parse_time:.3f}s total ({cached_count} cached, {parsed_count} parsed)" + ) # Create unified model with all records cache_ref_summary = f"per-test-base: {len(cache_refs)} cache files" From ae6b755bf2edae0f79a04cf4c53e2b64b2e45840 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:43:31 +0200 Subject: [PATCH 0249/1072] [caliper] orchestration: postprocess: show the stack trace --- projects/caliper/orchestration/postprocess.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 898a9929c..1cdee02f0 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1131,7 +1131,10 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") elif status == "failed": error_msg = result.get("error", "unknown error") + traceback_msg = result.get("traceback") logger.error(f"Step '{step_name}' failed: {error_msg}") + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) return True return False @@ -1233,7 +1236,10 @@ def _run_parse_step(self) -> None: error_msg = (status_data or {}).get( "error", f"Command failed with exit code {result.returncode}" ) + traceback_msg = (status_data or {}).get("traceback") logger.error("Caliper parse failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "parse", { @@ -1347,7 +1353,10 @@ def _run_visualize_step(self) -> None: error_msg = (status_data or {}).get( "error", f"Command failed with exit code {result.returncode}" ) + traceback_msg = (status_data or {}).get("traceback") logger.error("Caliper visualize failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "visualize", { From ec6b56d2c30d15f5efac4facd4201f3a30ebe7a2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 16:25:41 +0200 Subject: [PATCH 0250/1072] [caliper] improve the --include-filter behavior --- projects/caliper/cli/commands.py | 19 +++ projects/caliper/engine/parse.py | 4 +- projects/caliper/engine/traverse.py | 192 ++++++++++++++++++++++++++- projects/caliper/engine/visualize.py | 21 ++- 4 files changed, 225 insertions(+), 11 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 181d8c03b..c18a8b639 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -108,6 +108,11 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) +@click.option( + "--include-label", + multiple=True, + help="Include only test directories with matching labels (format: key=value).", +) @click.option( "--status-file", type=click.Path(path_type=Path), @@ -120,6 +125,7 @@ def parse_cmd( no_cache: bool, cache_dir: Path | None, show_matrix: bool, + include_label: tuple[str, ...], status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, @@ -135,6 +141,18 @@ def parse_cmd( mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + # Parse include-label options into filter dict + label_filter = None + if include_label: + label_filter = {} + for label_spec in include_label: + if "=" not in label_spec: + raise click.BadParameter( + f"Invalid label format '{label_spec}'. Use key=value format." + ) + key, value = label_spec.split("=", 1) + label_filter[key.strip()] = value.strip() + status = {"success": False} try: @@ -144,6 +162,7 @@ def parse_cmd( plugin=plugin, use_cache=not no_cache, show_parameter_matrix=show_matrix, + label_filter=label_filter, ) # Extract test directories from test nodes diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index 3a81f8fd0..368f712cb 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -5,6 +5,7 @@ import logging import time from pathlib import Path +from typing import Any from projects.caliper.engine.cache import ( cache_path_for_test_base, @@ -33,6 +34,7 @@ def run_parse( use_cache: bool, force_report_partial: bool = True, show_parameter_matrix: bool = True, + label_filter: dict[str, Any] | None = None, ) -> UnifiedRunModel: """ Run full parse or load valid cache. @@ -42,7 +44,7 @@ def run_parse( base_dir = base_dir.resolve() # Discover test bases - nodes = discover_test_bases(base_dir) + nodes = discover_test_bases(base_dir, label_filter=label_filter) # Always use per-test-base caching all_records = [] diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index 461e4f7d6..fa0214c41 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -14,8 +14,15 @@ MATRIXBENCHMARKING_MARKER = "settings.yaml" -def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: - """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode.""" +def discover_test_bases( + base_dir: Path, *, label_filter: dict[str, Any] | None = None +) -> list[TestBaseNode]: + """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode. + + Args: + base_dir: Base directory to search + label_filter: Optional dict of label key-value pairs that must match for inclusion + """ base_dir = base_dir.resolve() if not base_dir.is_dir(): raise FileNotFoundError(f"Base directory does not exist: {base_dir}") @@ -33,9 +40,10 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: path = Path(dirpath) marker_path = path / marker_found - labels = _load_labels( - marker_path, is_matrixbenchmarking=(marker_found == MATRIXBENCHMARKING_MARKER) - ) + # Apply label filtering if specified + if label_filter is not None and not _matches_label_filter(labels, label_filter): + continue + nodes.append( TestBaseNode( directory=path, @@ -47,6 +55,180 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: return sorted(nodes, key=lambda n: str(n.directory)) +def _load_hierarchical_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge __test_labels__.yaml files hierarchically from base_dir down to test_dir. + + Merges in order: + 1. base_dir/__test_labels__.*.yaml (all variants) + 2. parent_dir/__test_labels__.*.yaml (all variants) + 3. test_dir/__test_labels__.*.yaml (all variants) + 4. test_dir/__test_labels__.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main __test_labels__.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge __test_labels__.*.yaml files (excluding plain __test_labels__.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all __test_labels__.*.yaml files (but not __test_labels__.yaml itself) + pattern = str(dir_path / "__test_labels__.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=False) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main __test_labels__.yaml from the test directory (final priority) + main_labels_path = test_dir / MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=False) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _load_hierarchical_labels_matrixbenchmarking(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge settings.yaml files hierarchically from base_dir down to test_dir for MatrixBenchmarking. + + Merges in order: + 1. base_dir/settings.*.yaml (all variants) + 2. parent_dir/settings.*.yaml (all variants) + 3. test_dir/settings.*.yaml (all variants) + 4. test_dir/settings.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main settings.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge settings.*.yaml files (excluding plain settings.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all settings.*.yaml files (but not settings.yaml itself) + pattern = str(dir_path / "settings.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=True) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main settings.yaml from the test directory (final priority) + main_labels_path = test_dir / MATRIXBENCHMARKING_MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=True) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _matches_label_filter(labels: dict[str, Any], label_filter: dict[str, Any]) -> bool: + """Check if labels match the specified filter criteria. + + Args: + labels: The hierarchically merged labels from the test directory + label_filter: Dict of key-value pairs that must match + + Returns: + True if all filter criteria match, False otherwise + + Examples: + _matches_label_filter({"labels": {"platform": "CKS", "version": "3.5"}}, + {"platform": "CKS"}) -> True + _matches_label_filter({"labels": {"platform": "OCP"}}, + {"platform": "CKS"}) -> False + """ + # Extract the labels section if it exists (for __test_labels__.yaml structure) + actual_labels = labels.get("labels", {}) if isinstance(labels.get("labels"), dict) else labels + + for filter_key, filter_value in label_filter.items(): + # Support nested key access with dot notation (e.g., "labels.platform") + if "." in filter_key: + keys = filter_key.split(".") + current = labels + for key in keys: + if not isinstance(current, dict) or key not in current: + return False + current = current[key] + if current != filter_value: + return False + else: + # Direct key access in labels section or root level + if filter_key not in actual_labels and filter_key not in labels: + return False + value = actual_labels.get(filter_key, labels.get(filter_key)) + if value != filter_value: + return False + + return True + + +def _deep_merge_dict(target: dict[str, Any], source: dict[str, Any]) -> None: + """Deep merge source dict into target dict, with source values taking precedence.""" + for key, value in source.items(): + if key in target and isinstance(target[key], dict) and isinstance(value, dict): + _deep_merge_dict(target[key], value) + else: + target[key] = value + + def _load_labels(path: Path, is_matrixbenchmarking: bool = False) -> dict[str, Any]: """Load labels from either __test_labels__.yaml or MatrixBenchmarking settings.yaml.""" raw = path.read_text(encoding="utf-8") diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index e2947c151..529a3d03d 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -70,19 +70,30 @@ def run_visualize( use_cache: bool, cache_path: Path | None, ) -> list[str]: + # Convert include pairs to directory-level filter for discovery + directory_filter = None + if include_pairs: + inc = parse_filter_kv(include_pairs) + directory_filter = inc # Use include filters for directory discovery + + # Apply directory-level filtering during parsing model = run_parse( base_dir=base_dir, plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, + label_filter=directory_filter, ) + + # Still apply record-level filtering for exclude filters and any remaining include filters inc = parse_filter_kv(include_pairs) exc = parse_filter_kv(exclude_pairs) - model.unified_result_records = filter_records( - model.unified_result_records, - include=inc, - exclude=exc, - ) + if exc: # Only filter records if there are exclude filters + model.unified_result_records = filter_records( + model.unified_result_records, + include={}, # Include filtering already done at directory level + exclude=exc, + ) cfg = resolve_visualize_config(base_dir, visualize_config_path) ids = resolve_report_ids( reports_csv=reports_csv, From 9283f387b41bd8d9667e961b8fac8dedf3ecb57d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 11:54:53 +0200 Subject: [PATCH 0251/1072] Revert "[caliper] remove unnecessary reference to the hierachical format" This reverts commit 5b392f0d2f9597830f38ff17ff2b77b912e5b6a4. --- projects/caliper/cli/commands.py | 6 +- projects/caliper/engine/kpi/analyze.py | 349 +----------------- .../engine/kpi/analyze_hierarchical.py | 214 +++++++++++ projects/caliper/orchestration/postprocess.py | 59 +-- .../caliper/orchestration/step_logging.py | 348 +++++++++++++++++ 5 files changed, 622 insertions(+), 354 deletions(-) create mode 100644 projects/caliper/engine/kpi/analyze_hierarchical.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index c18a8b639..a6937755c 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -639,9 +639,9 @@ def kpi_analyze( output: Path, plugin_module: str, ) -> None: - """Analyze KPIs for regression.""" + """Analyze KPIs for regressions using hierarchical format.""" try: - from projects.caliper.engine.kpi.analyze import find_baseline_kpis + from projects.caliper.engine.kpi.analyze_hierarchical import find_most_recent_baseline from projects.caliper.engine.load_plugin import load_plugin # Load plugin for KPI definitions and analysis rules @@ -652,7 +652,7 @@ def kpi_analyze( click.echo(f"🔌 Using plugin: {plugin_module}") - baseline_kpis = find_baseline_kpis(baseline_dir) + baseline_kpis = find_most_recent_baseline(baseline_dir) if not baseline_kpis: click.echo( f"❌ No kpis.json files found in baseline directory: {baseline_dir}", err=True diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 47ae439ff..246c317e5 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,343 +2,32 @@ from __future__ import annotations -import json -import logging -import time -from pathlib import Path from typing import Any -logger = logging.getLogger(__name__) - - -def run_kpi_analysis( - current_kpi_file: Path, - historical_data_dir: Path, - output_file: Path, - plugin_module: str, -) -> int: - """Run KPI analysis and generate output file. - - Args: - current_kpi_file: Path to current KPI JSON file - historical_data_dir: Directory containing historical KPI files - output_file: Path where analysis results will be written - plugin_module: Plugin module name for analysis - - Returns: - Exit code (0 for success, non-zero for failure) - """ - try: - logger.info(f"Running KPI analysis for: {current_kpi_file}") - logger.info(f"Historical data directory: {historical_data_dir}") - logger.info(f"Output file: {output_file}") - logger.info(f"Plugin module: {plugin_module}") - - # Check if current KPI file exists - if not current_kpi_file.exists(): - logger.error(f"Current KPI file not found: {current_kpi_file}") - return 1 - - # Check if historical data directory exists - if not historical_data_dir.exists(): - logger.error(f"Historical data directory not found: {historical_data_dir}") - return 1 - - # Find all historical KPI files - historical_kpi_files = [] - for kpi_file in historical_data_dir.rglob("kpis.json"): - historical_kpi_files.append(str(kpi_file)) - logger.info(f"Found historical KPI file: {kpi_file}") - - if not historical_kpi_files: - logger.warning("No historical KPI files found for analysis") - # Return special exit code 2 to indicate warning (no historical data) - analysis_result = { - "status": "warning", - "message": "no historical KPI found for regression testing", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": 0, - "plugin_module": plugin_module, - "completed_at": time.time(), - } - # Write warning result to output file - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - return 2 # Special exit code for warning - - logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") - - # Create analysis result - analysis_result = { - "status": "success (stub)", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": len(historical_kpi_files), - "plugin_module": plugin_module, - "completed_at": time.time(), - } - - # Create output directory if needed - output_file.parent.mkdir(parents=True, exist_ok=True) - - # Write analysis results to output file - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - f.write("\n") # Add EOL at EOF - - logger.info(f"Analysis completed successfully, results written to: {output_file}") - return 0 - - except Exception as e: - logger.exception("KPI analysis failed") - - # Create failure result - failure_result = { - "status": "failed", - "error": str(e), - "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, - "completed_at": time.time(), - } - - try: - # Attempt to write failure result - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(failure_result, f, indent=2) - f.write("\n") # Add EOL at EOF - except Exception: - logger.exception("Failed to write error result to output file") - - return 1 - - -def analyze_kpis( - postprocess_config, # CaliperOrchestrationPostprocessConfig - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Run KPI analysis step and return result status. - - This is the orchestration interface for KPI analysis. - """ - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled"} - - # Determine paths for analysis - analyze_config = postprocess_config.analyze - - # Historical KPIs directory path - historical_kpis_dir = Path(analyze_config.historical_kpis) - if not historical_kpis_dir.is_absolute(): - historical_kpis_dir = output_dir / historical_kpis_dir - - # Output path for analysis results - output_path = output_dir / analyze_config.output - - try: - # Check if required files exist - if not current_kpis_file.exists(): - return { - "status": "failed", - "error": f"Current KPI file not found: {current_kpis_file}", - "completed_at": time.time(), - } - - if not historical_kpis_dir.exists(): - return { - "status": "failed", - "error": f"Historical KPIs directory not found: {historical_kpis_dir}", - "completed_at": time.time(), - } - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=current_kpis_file, - historical_data_dir=historical_kpis_dir, - output_file=output_path, - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read the results from the output file - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return success with output file path for notifications - return { - "status": "success", - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": "Analysis completed but no output file was generated", - "completed_at": time.time(), - } - elif exit_code == 2: - # Warning: no historical data found - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return warning status with message - return { - "status": "warning", - "message": result_data.get( - "message", "no historical KPI found for regression testing" - ), - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "baseline_files_count": 0, - "completed_at": time.time(), - } - else: - # Read error from output file if it exists - error_msg = f"Analysis failed with exit code {exit_code}" - if output_path.exists(): - try: - with open(output_path) as f: - result_data = json.load(f) - error_msg = result_data.get("error", error_msg) - except Exception: - pass - - return { - "status": "failed", - "error": error_msg, - "completed_at": time.time(), - } - - except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Analysis step failed with {error_msg}") - - # Don't suppress the exception - provide detailed error info and re-raise - import traceback - - full_traceback = traceback.format_exc() - logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") - - # Re-raise the original exception so the caller can see the full details - raise RuntimeError(f"KPI analysis failed: {error_msg}") from e - - -def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: - """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. - - Args: - historical_dir: Directory to search for historical kpis.json files - - Returns: - Dictionary mapping file paths to loaded KPI JSON objects - """ - baseline_kpis = {} - kpi_files = list(historical_dir.rglob("kpis.json")) - - if not kpi_files: - logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") - return baseline_kpis - - logger.info(f"Found {len(kpi_files)} historical KPI files to load") - - for kpi_file in kpi_files: - try: - with open(kpi_file) as f: - kpi_data = json.load(f) - - # Validate that it's a hierarchical format (schema_version 2) - schema_version = kpi_data.get("schema_version", "unknown") - if schema_version != "2": - logger.warning( - f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" - ) - continue - - baseline_kpis[kpi_file] = kpi_data - logger.debug(f"Loaded KPI file: {kpi_file}") - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON in {kpi_file}: {e}") - continue - except FileNotFoundError as e: - logger.error(f"KPI file not found: {kpi_file}: {e}") - continue - except Exception as e: - logger.error(f"Failed to load KPI file {kpi_file}: {e}") - continue - - logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") - return baseline_kpis +from projects.caliper.engine.model import RegressionFinding def run_analyze( *, current_path: Any, - baseline_kpis: dict[Path, dict[str, Any]], + baseline_path: Any, output_path: Any, plugin: Any = None, -) -> dict[str, Any]: - """Run KPI analysis against ALL baseline files (CLI interface).""" - # For CLI use, we need to extract the baseline directory from the dict - # This is a bit of a hack, but the CLI interface expects a directory - if baseline_kpis: - # Get the parent directory of the first baseline file - first_baseline_path = next(iter(baseline_kpis.keys())) - historical_dir = first_baseline_path.parent - - # Extract plugin module name if available - plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=Path(current_path), - historical_data_dir=historical_dir, - output_file=Path(output_path), - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read results and convert to expected format for CLI - try: - with open(Path(output_path)) as f: - result_data = json.load(f) - return { - "status": "success", - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": result_data.get("completed_at", time.time()), - } - except Exception as e: - return { - "status": "failed", - "error": f"Failed to read analysis results: {e}", - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } +) -> list[RegressionFinding]: + """Run KPI analysis - hierarchical format only.""" + from pathlib import Path + + from .analyze_hierarchical import analyze_hierarchical_kpis + + result = analyze_hierarchical_kpis( + current_kpis_path=Path(current_path), + baseline_kpis_path=Path(baseline_path), + output_path=Path(output_path), + plugin=plugin, + ) + + # Convert to expected format for CLI + if result["status"] == "success": + return [] # Return empty list for now, actual findings are in the output file else: - return { - "status": "failed", - "error": "No baseline KPI files provided", - "completed_at": time.time(), - } - - -# EOF + raise RuntimeError(f"Analysis failed: {result.get('error', 'Unknown error')}") diff --git a/projects/caliper/engine/kpi/analyze_hierarchical.py b/projects/caliper/engine/kpi/analyze_hierarchical.py new file mode 100644 index 000000000..c76a9514d --- /dev/null +++ b/projects/caliper/engine/kpi/analyze_hierarchical.py @@ -0,0 +1,214 @@ +"""KPI analysis for hierarchical format.""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def analyze_hierarchical_kpis( + current_kpis_path: Path, + baseline_kpis_path: Path, + output_path: Path, + plugin: Any = None, +) -> dict[str, Any]: + """ + Analyze hierarchical KPI format for regressions. + + Args: + current_kpis_path: Path to current KPIs JSON file (hierarchical format) + baseline_kpis_path: Path to baseline KPIs JSON file (hierarchical format) + output_path: Path where analysis results will be written + plugin: Caliper plugin instance for KPI definitions and analysis rules + + Returns: + Analysis result dictionary with status, findings, etc. + """ + try: + # Load KPI files + with open(current_kpis_path) as f: + current_data = json.load(f) + + with open(baseline_kpis_path) as f: + baseline_data = json.load(f) + + # Validate format + if not isinstance(current_data, dict) or not isinstance(baseline_data, dict): + return { + "status": "failed", + "error": "KPI files must be in hierarchical format (dict), not list format", + "completed_at": time.time(), + } + + # Extract schema version and validate + current_schema = current_data.get("schema_version", "unknown") + baseline_schema = baseline_data.get("schema_version", "unknown") + + if current_schema != "2" or baseline_schema != "2": + return { + "status": "failed", + "error": f"Only schema_version 2 supported (current: {current_schema}, baseline: {baseline_schema})", + "completed_at": time.time(), + } + + # Analyze metrics using plugin-aware logic + current_metrics = current_data.get("metrics", {}) + baseline_metrics = baseline_data.get("metrics", {}) + + findings = [] + regressions = 0 + improvements = 0 + + # Get plugin-specific KPI definitions if available + if plugin and hasattr(plugin, "compute_kpis"): + try: + # Try to get KPI metadata from plugin + # This is a stub - plugins may expose KPI definitions differently + logger.debug(f"Plugin available for analysis: {plugin.__class__.__name__}") + except Exception as e: + logger.warning(f"Could not get KPI definitions from plugin: {e}") + + # Compare common metrics + for metric_name in current_metrics: + if metric_name in baseline_metrics: + current_metric = current_metrics[metric_name] + baseline_metric = baseline_metrics[metric_name] + + current_value = current_metric.get("value") + baseline_value = baseline_metric.get("value") + + if current_value is not None and baseline_value is not None: + try: + curr_val = float(current_value) + base_val = float(baseline_value) + + # Get direction preference from metric metadata or use default + higher_is_better = current_metric.get("higher_is_better", False) + if "labels" in current_metric: + higher_is_better = current_metric["labels"].get( + "higher_is_better", False + ) + + # Calculate change + change_percent = ( + ((curr_val - base_val) / base_val) * 100 + if base_val != 0 + else float("inf") + ) + + # Default threshold: 5% change (can be made configurable via plugin) + threshold_percent = 5.0 + + # Determine if this is a regression based on direction + is_regression = False + is_improvement = False + + if abs(change_percent) > threshold_percent: + if higher_is_better: + # For metrics where higher is better + if curr_val < base_val: # Decreased + is_regression = True + elif curr_val > base_val: # Increased + is_improvement = True + else: + # For metrics where lower is better + if curr_val > base_val: # Increased + is_regression = True + elif curr_val < base_val: # Decreased + is_improvement = True + + if is_regression: + regressions += 1 + findings.append( + { + "metric": metric_name, + "type": "regression", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + elif is_improvement: + improvements += 1 + findings.append( + { + "metric": metric_name, + "type": "improvement", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + + except (TypeError, ValueError): + logger.warning( + f"Could not compare metric {metric_name}: non-numeric values" + ) + + # Create analysis results + analysis_result = { + "analysis_timestamp": time.time(), + "current_file": str(current_kpis_path), + "baseline_file": str(baseline_kpis_path), + "schema_version": "2", + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "findings": findings, + "summary": f"Found {regressions} regressions and {improvements} improvements across {len([m for m in current_metrics if m in baseline_metrics])} metrics", + } + + # Write results + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(analysis_result, f, indent=2) + + logger.info(f"Analysis completed: {regressions} regressions, {improvements} improvements") + + return { + "status": "success", + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "output_file": str(output_path), + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "completed_at": time.time(), + } + + except FileNotFoundError as e: + return { + "status": "failed", + "error": f"File not found: {e}", + "completed_at": time.time(), + } + except json.JSONDecodeError as e: + return { + "status": "failed", + "error": f"Invalid JSON format: {e}", + "completed_at": time.time(), + } + except Exception as e: + logger.exception("Analysis failed") + return { + "status": "failed", + "error": f"Analysis failed: {e}", + "completed_at": time.time(), + } + + +def find_most_recent_baseline(historical_dir: Path) -> Path | None: + """Find the most recently modified kpis.json file in historical directory.""" + kpi_files = list(historical_dir.rglob("kpis.json")) + if not kpi_files: + return None + return max(kpi_files, key=lambda p: p.stat().st_mtime) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 1cdee02f0..eef73c57d 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -21,7 +21,6 @@ from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, - build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -896,23 +895,35 @@ def _run_analyse_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "analyse_kpis_status.yaml" - # Build CLI command - command = build_analyse_kpis_command( - config=postprocess_config, - tree_root=base_dir, - manifest_path=manifest_path, - status_file=status_file, - output_file=output_file, - current_kpis_file=current_kpis_file, - historical_kpis_dir=historical_kpis_dir, + # Note: CLI command building removed in favor of direct function call + + # Load plugin for KPI definitions + from projects.caliper.engine.load_plugin import load_plugin + + plugin = load_plugin(plugin_module) + + # Find the most recent baseline file + from projects.caliper.engine.kpi.analyze_hierarchical import ( + analyze_hierarchical_kpis, + find_most_recent_baseline, ) - # Execute command using generic function - result, status_data, log_file = _execute_caliper_command( - command=command, - step_name="caliper kpi analyse-kpis", - status_file=status_file, - step_logs_dir=step_logs_dir, + baseline_file = find_most_recent_baseline(historical_kpis_dir) + if not baseline_file: + return { + "status": "failed", + "error": f"No baseline KPI files found in {historical_kpis_dir}", + "completed_at": time.time(), + "log_file": None, + } + + # Run hierarchical KPI analysis using core engine + logger.info(f"Running KPI analysis: {current_kpis_file} vs {baseline_file}") + result = analyze_hierarchical_kpis( + current_kpis_path=current_kpis_file, + baseline_kpis_path=baseline_file, + output_path=output_file, + plugin=plugin, ) # Clean up temporary status file @@ -921,20 +932,26 @@ def _run_analyse_kpis( except FileNotFoundError: pass - # Convert to expected format - if status_data.get("success"): + # Handle the analysis result + if result["status"] == "success": + logger.info( + f"Analysis completed: {result['regressions_count']} regressions, {result['improvements_count']} improvements" + ) return { "status": "success", "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "findings_count": result["findings_count"], + "regressions_count": result["regressions_count"], + "improvements_count": result["improvements_count"], "completed_at": time.time(), - "log_file": log_file, + "log_file": None, } else: return { "status": "failed", - "error": status_data.get("error", "Unknown error"), + "error": result.get("error", "Analysis failed"), "completed_at": time.time(), - "log_file": log_file, + "log_file": None, } except Exception as e: diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 66cd83137..0febd3ecb 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -11,6 +11,7 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path +from typing import Any # Thread-local storage for step-specific handlers _step_local_handlers = threading.local() @@ -234,3 +235,350 @@ def cleanup_step_logging(): if hasattr(_step_local_handlers, "file_handler"): _step_local_handlers.file_handler.close() del _step_local_handlers.file_handler + + +def _log_command_banner( + step_name: str, + command_line: str, + description: str = "", + step_args: dict[str, Any] | None = None, +) -> None: + """ + Log a command reproduction banner at the start of a step. + + Args: + step_name: Name of the step (e.g., "caliper_parse") + command_line: The CLI command to reproduce this step + description: Brief description of what this step does + step_args: Dictionary of step arguments for reference + """ + logger = logging.getLogger(__name__) + + LINE_WIDTH = 80 + + logger.info("") + logger.info("=" * LINE_WIDTH) + logger.info(f"CALIPER STEP: {step_name.upper()}") + if description: + logger.info(f"DESCRIPTION: {description}") + logger.info("=" * LINE_WIDTH) + logger.info(f"COMMAND TO REPRODUCE:\n{command_line}") + + if step_args: + logger.info("") + logger.info("STEP PARAMETERS:") + for key, value in step_args.items(): + if value is not None: + # Truncate very long values + value_str = str(value) + if len(value_str) > 100: + value_str = value_str[:97] + "..." + logger.info(f" {key}: {value_str}") + + logger.info("=" * LINE_WIDTH) + logger.info("") + + +def log_parse_command( + base_dir: Path, + plugin_module: str, + use_cache: bool, + manifest_path: Path | None = None, +) -> None: + """Log the CLI command to reproduce the parse step.""" + cache_flag = "" if use_cache else " --no-cache" + command = f'caliper parse --base-dir "{base_dir}" --plugin {plugin_module}{cache_flag}' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "use_cache": use_cache, + } + + # Only include manifest_path if it's provided and safe to log + if manifest_path: + step_args["manifest_path"] = str(manifest_path) + + _log_command_banner( + "caliper_parse", + command, + "Parse test results from artifact tree into unified model", + step_args, + ) + + +def log_visualize_command( + base_dir: Path, + plugin_module: str, + output_dir: Path, + reports_csv: str | None = None, + report_group: str | None = None, + visualize_config_path: Path | None = None, + include_pairs: tuple[str, ...] = (), + exclude_pairs: tuple[str, ...] = (), + use_cache: bool = True, +) -> None: + """Log the CLI command to reproduce the visualize step.""" + command_parts = [ + "caliper visualize", + f'--base-dir "{base_dir}"', + f'--output-dir "{output_dir}"', + f"--plugin {plugin_module}", + ] + + if not use_cache: + command_parts.append("--no-cache") + if reports_csv: + command_parts.append(f'--reports "{reports_csv}"') + if report_group: + command_parts.append(f'--report-group "{report_group}"') + if visualize_config_path: + command_parts.append(f'--visualize-config "{visualize_config_path}"') + + for include_pair in include_pairs: + command_parts.append(f'--include "{include_pair}"') + for exclude_pair in exclude_pairs: + command_parts.append(f'--exclude "{exclude_pair}"') + + command = " \\\n ".join(command_parts) + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_dir": str(output_dir), + "use_cache": use_cache, + } + + # Only include optional parameters if they're provided + if reports_csv: + step_args["reports_csv"] = reports_csv + if report_group: + step_args["report_group"] = report_group + if visualize_config_path: + step_args["visualize_config_path"] = str(visualize_config_path) + if include_pairs: + step_args["include_pairs"] = list(include_pairs) + if exclude_pairs: + step_args["exclude_pairs"] = list(exclude_pairs) + + _log_command_banner( + "caliper_visualize", + command, + "Generate visualization reports and plots from parsed data", + step_args, + ) + + +def log_artifacts_to_kpis_command( + base_dir: Path, + plugin_module: str, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the KPI generation step.""" + command = f'caliper kpi generate --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_kpi_generate", command, "Generate KPI metrics from parsed test results", step_args + ) + + +def log_kpi_export_command( + input_path: Path, + target_system: str = "opensearch", +) -> None: + """Log the CLI command to reproduce the KPI export step.""" + command = f'caliper kpi export --input "{input_path}" --target {target_system}' + + step_args = { + "input_path": str(input_path), + "target_system": target_system, + } + + _log_command_banner( + "caliper_kpi_export", command, "Export KPI metrics to external monitoring system", step_args + ) + + +def log_ai_data_command( + base_dir: Path, + plugin_module: str, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the AI evaluation export step.""" + command = f'caliper ai-eval export --base-dir "{base_dir}" --plugin {plugin_module} --output "{output_file}"' + + step_args = { + "base_dir": str(base_dir), + "plugin_module": plugin_module, + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_ai_data_export", + command, + "Export AI evaluation payload with structured test directories and artifacts", + step_args, + ) + + +def log_s3_import_command( + bucket: str, + prefix: str, + output_dir: Path, +) -> None: + """Log the CLI command to reproduce the S3 import step.""" + command = ( + f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' + ) + + step_args = { + "bucket": bucket, + "prefix": prefix, + "output_dir": str(output_dir), + } + + _log_command_banner( + "caliper_s3_import", + command, + "Import historical data from S3 for analysis", + step_args, + ) + + +def log_s3_export_command( + bucket: str, + export_path: str, + from_dir: Path, + include_csv: bool, + include_kpis_json: bool, + include_ai_data: bool, +) -> None: + """Log the CLI command to reproduce the S3 export step.""" + command_parts = [f'caliper s3-export --from-dir "{from_dir}" --bucket "{bucket}"'] + + if export_path: + # Parse the export path to get prefix, instance, directory + parts = export_path.split("/") + if len(parts) >= 3: + instance = parts[0] + directory = parts[1] + prefix = "/".join(parts[2:]) if len(parts) > 2 else "" + command_parts.append(f'--instance "{instance}"') + command_parts.append(f'--directory "{directory}"') + if prefix: + command_parts.append(f'--prefix "{prefix}"') + + # Note: Include flags use default values (--include-csv, --include-kpis-json, --include-ai-data all default to True) + + command = " ".join(command_parts) + + step_args = { + "bucket": bucket, + "export_path": export_path, + "from_dir": str(from_dir), + "include_csv": include_csv, + "include_kpis_json": include_kpis_json, + "include_ai_data": include_ai_data, + } + + _log_command_banner( + "caliper_s3_export", + command, + "Export postprocess artifacts to S3", + step_args, + ) + + +def log_analyse_kpis_command( + current_kpi_file: Path, + historical_dir: Path, + output_file: Path, +) -> None: + """Log the CLI command to reproduce the KPI analysis step.""" + command = f'caliper analyse-kpis --current "{current_kpi_file}" --historical-dir "{historical_dir}" --output "{output_file}"' + + step_args = { + "current_kpi_file": str(current_kpi_file), + "historical_dir": str(historical_dir), + "output_file": str(output_file), + } + + _log_command_banner( + "caliper_analyse_kpis", + command, + "Analyze current KPIs against historical data", + step_args, + ) + + +def log_analyze_command( + base_dir: Path, + plugin_module: str, + current_kpis_path: Path | None = None, + baseline_file: Path | None = None, + historical_kpis_dir: Path | None = None, + output_path: Path | None = None, +) -> None: + """Log the CLI command to reproduce the analyze step.""" + # Use the correct command structure: caliper kpi analyze + command = "caliper kpi analyze" + + # Add current KPI file path if provided + if current_kpis_path: + command += f' --current "{current_kpis_path}"' + + # Add the historical directory, not the specific file + if historical_kpis_dir: + command += f' --baseline-dir "{historical_kpis_dir}"' + + # Add output path if provided + if output_path: + command += f' --output "{output_path}"' + + # Add plugin module + command += f" --plugin {plugin_module}" + + # Add helpful context + command += "\n# Analyzes hierarchical KPI format for regressions" + command += "\n# The tool will automatically find the most recent kpis.json file in the baseline directory" + command += "\n# To re-run this analysis step independently, use the command above" + command += f"\n# Plugin: {plugin_module}" + command += f"\n# Base dir: {base_dir}" + if baseline_file: + command += f"\n# Most recent baseline file found: {baseline_file}" + + step_args = { + "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, + "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, + "baseline_file_selected": str(baseline_file) if baseline_file else None, + "output_path": str(output_path) if output_path else None, + "plugin_module": plugin_module, + "base_dir": str(base_dir), + } + + _log_command_banner( + "caliper_analyze", command, "Run regression analysis on test results", step_args + ) + + +def log_kpis_to_csv_command( + input_path: Path, + output_path: Path, +) -> None: + """Log the CLI command to reproduce the KPI CSV export step.""" + command = f'caliper kpi csv-export --input "{input_path}" --output "{output_path}"' + + step_args = { + "input_path": str(input_path), + "output_path": str(output_path), + } + + _log_command_banner( + "caliper_kpi_csv_export", command, "Export KPI metrics to CSV format", step_args + ) From 9901b0baf5ef3d30359946a5105e3b6a1fb20c6c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 16:26:49 +0200 Subject: [PATCH 0252/1072] [caliper] engine: traverse: include hierarchical label parsing --- projects/caliper/engine/traverse.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index fa0214c41..add739209 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -39,7 +39,18 @@ def discover_test_bases( continue path = Path(dirpath) - marker_path = path / marker_found + + # Use hierarchical label loading for both marker types + if marker_found == MARKER: + labels = _load_hierarchical_labels(path, base_dir) + else: + # Use hierarchical loading for MatrixBenchmarking settings.yaml too + labels = _load_hierarchical_labels_matrixbenchmarking(path, base_dir) + + # Skip directory if skip: true is set in labels + if labels.get("skip") is True: + continue + # Apply label filtering if specified if label_filter is not None and not _matches_label_filter(labels, label_filter): continue From 802a178ddaa3b711500c5a9c2f7deb8eb055ae69 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 11:27:36 -0400 Subject: [PATCH 0253/1072] fix: add init container to fix PVC permissions for GuideLLM The /results PVC is mounted with root ownership, but OpenShift runs the GuideLLM container as a non-root UID. Add an init container that chmod's the mount so GuideLLM can write benchmark results. Co-authored-by: Cursor --- .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8f1ef15b6..8ca1f3253 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -17,6 +17,8 @@ spec: spec: serviceAccountName: default restartPolicy: Never + securityContext: + fsGroup: 0 containers: - name: guidellm image: {{ image }} From 27be386a96d80498e760e0495a068aab14825fc6 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 13:15:14 -0400 Subject: [PATCH 0254/1072] fix: make fsGroup opt-in for GuideLLM job pods fsGroup: 0 is incompatible with restricted/nonroot SCCs on shared clusters. Move it behind an optional benchmarks.guidellm.fs_group config key (disabled by default). Clusters where the CSI driver provisions root-owned volumes can opt in via configOverrides. Co-authored-by: Cursor --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 3 +++ .../run_guidellm_benchmark/templates/guidellm_job.yaml.j2 | 4 +++- projects/guidellm/toolbox/run_guidellm_benchmark/utils.py | 5 +++++ projects/rhaiis/orchestration/test_phase.py | 3 +++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index fa82ff1ec..17b51493c 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -34,6 +34,7 @@ def run( pvc_size: str = "1Gi", guidellm_args: list[str] | None = None, hf_token_secret: str = "", + fs_group: int | None = None, ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -47,6 +48,7 @@ def run( pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds securityContext.fsGroup to the GuideLLM job pod. """ execute_tasks(locals()) @@ -142,6 +144,7 @@ def create_guidellm_resources_task(args, ctx): endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, hf_token_secret=args.hf_token_secret, + fs_group=args.fs_group, ), ) return f"GuideLLM benchmark {ctx.benchmark_name} created" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8ca1f3253..10b40fe0a 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -17,8 +17,10 @@ spec: spec: serviceAccountName: default restartPolicy: Never +{% if fs_group is not none %} securityContext: - fsGroup: 0 + fsGroup: {{ fs_group }} +{% endif %} containers: - name: guidellm image: {{ image }} diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 7cab6a2c2..e3ef5a0a9 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -176,6 +176,7 @@ def render_guidellm_job_from_parts( endpoint_url: str, guidellm_args: list[str], hf_token_secret: str = "", + fs_group: int | None = None, ) -> dict[str, Any]: """Render a GuideLL-M job manifest from individual components. @@ -186,6 +187,9 @@ def render_guidellm_job_from_parts( endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds a pod-level securityContext.fsGroup to ensure + the PVC is writable by the container. Needed on clusters where the + CSI driver provisions volumes with root-only permissions. Returns: Job manifest as dict @@ -198,6 +202,7 @@ def render_guidellm_job_from_parts( "name": name, "image": image, "hf_token_secret": hf_token_secret, + "fs_group": fs_group, }, ) manifest = yaml.safe_load(rendered_yaml) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 3ffbf7529..d630519f9 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -435,6 +435,7 @@ def _run_workload_benchmark( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) @@ -655,6 +656,7 @@ def _run_warmup_step( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) logger.info("Warmup completed") except Exception: @@ -720,6 +722,7 @@ def _run_profiler_step( pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), guidellm_args=guidellm_args, hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) finally: enable_profiler_gate( From a5de6caa3cf75f160096bc30bd70dff3be74fc19 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 14:14:58 -0400 Subject: [PATCH 0255/1072] chore: add missing trtllm_config defaults for Nemotron-H Add mamba_ssm_cache_dtype, mamba_ssm_stochastic_rounding, mamba_ssm_philox_rounds, enable_chunked_prefill, and num_postprocess_workers to the base trtllm_config so they can be overridden via FournosJob configOverrides. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index c88c057f3..89b904e48 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -41,10 +41,15 @@ engines: dtype: auto enable_block_reuse: false free_gpu_memory_fraction: 0.90 + mamba_ssm_cache_dtype: "" + mamba_ssm_stochastic_rounding: false + mamba_ssm_philox_rounds: 0 cuda_graph_config: enable_padding: false max_batch_size: 256 enable_attention_dp: false + enable_chunked_prefill: false + num_postprocess_workers: 0 moe_config: backend: CUTLASS stream_interval: 0 From f221ef179526acea205cce8455af8a11e31bc2c1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 15:35:58 -0400 Subject: [PATCH 0256/1072] fix: include trtllm_config in dashboard runtime_args The runtime_args field on the dashboard was only populated from engine_args, missing the trtllm_config entries (kv_cache, CUDA graphs, MoE, etc.). Thread trtllm_config through to _create_test_labels so it appears in the dashboard alongside engine args. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index d630519f9..b0e6c8bb6 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import logging import os import uuid as _uuid_mod @@ -281,6 +282,7 @@ def _run_test( _warnings.append("Profiler trace upload failed") # Phase 2: benchmark + post-processing for ALL workloads + trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None for wl_key in workload_keys: _run_workload_benchmark( model_key=model_key, @@ -299,6 +301,7 @@ def _run_test( run_uuid=run_uuid, version=version, cluster_tag=cluster_tag, + trtllm_config=trtllm_cfg, ) try: @@ -364,6 +367,7 @@ def _run_workload_benchmark( run_uuid: str, version: str, cluster_tag: str, + trtllm_config: dict | None = None, ) -> None: """Run benchmark and post-processing for a single workload. @@ -396,6 +400,7 @@ def _run_workload_benchmark( cluster_tag=cluster_tag, accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, + trtllm_config=trtllm_config, ) if not run_benchmark: @@ -451,9 +456,16 @@ def _create_test_labels( cluster_tag: str = "", accelerator_chip: str = "", run_uuid: str = "", + trtllm_config: dict | None = None, ) -> None: _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") - runtime_args = "; ".join(f"{k}: {v}" for k, v in engine_args.items()) + parts = [f"{k}: {v}" for k, v in engine_args.items()] + for key, value in (trtllm_config or {}).items(): + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, (dict, list)) else value + ) + parts.append(f"trtllm.{key}: {formatted_value}") + runtime_args = "; ".join(parts) tp = ( engine_args.get("tensor-parallel-size") or engine_args.get("tp-size") From c4d582e7a97fcc743eddb7299d545bb7cbdafe53 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 19:05:31 -0400 Subject: [PATCH 0257/1072] chore: use recommended sglang serve entrypoint Replace deprecated 'python -m sglang.launch_server' with the recommended 'sglang serve' command. Co-authored-by: Cursor --- projects/rhaiis/orchestration/manifests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py index c181288a4..caa96c821 100644 --- a/projects/rhaiis/orchestration/manifests.py +++ b/projects/rhaiis/orchestration/manifests.py @@ -220,7 +220,7 @@ def _build_vllm_sglang_container( env_vars_list: list[dict], ) -> dict[str, Any]: if engine == "sglang": - command = ["python3", "-m", "sglang.launch_server"] + command = ["sglang", "serve"] if storage_source == "hf": args = [f"--model-path={model_id}", "--port=8080", "--host=0.0.0.0"] else: From f88e6a7b57dc2a340056753f31fb4f1bba016db2 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 21:57:28 -0400 Subject: [PATCH 0258/1072] docs: update README with multi-engine support Add SGLang and TRT-LLM documentation: engine abstraction overview, per-engine FournosJob YAML examples, updated configOverrides table with new engine/deploy keys, and manifests.py in entrypoints list. Co-authored-by: Cursor --- projects/rhaiis/README.md | 171 +++++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 19 deletions(-) diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index 1ccc40af6..e59c8e29e 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -3,10 +3,11 @@ `rhaiis` is the Forge project for benchmarking AI inference engines on OpenShift using KServe InferenceService. -The current implementation focuses on single-replica vLLM deployments benchmarked -with GuideLLM. The workflow deploys an InferenceService, runs warmup and/or -profiler passes, executes multi-profile benchmarks, generates a dashboard CSV via -the Caliper postprocessing pipeline, syncs results to S3, and cleans up. +Supports **vLLM**, **SGLang**, and **TRT-LLM** engines via a generic abstraction. +The workflow deploys an InferenceService with the selected engine, runs warmup +and/or profiler passes, executes multi-profile GuideLLM benchmarks, generates a +dashboard CSV via the Caliper postprocessing pipeline, syncs results to S3, and +cleans up. ## Workflow sequence @@ -54,7 +55,7 @@ to the canonical runner. Config is split into `config.yaml` (base) and `config.d/` (per-domain): - [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper postprocessing -- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types, S3 settings, profiler +- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, engine config (vLLM/SGLang/TRT-LLM), images, gpu_types, S3 settings, profiler - [`orchestration/config.d/models.yaml`](./orchestration/config.d/models.yaml) — 52 model definitions - [`orchestration/config.d/workloads.yaml`](./orchestration/config.d/workloads.yaml) — workload profiles @@ -62,15 +63,44 @@ Key sections: | Section | Purpose | |---------|---------| -| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args, S3 config | +| `rhaiis` | Namespace, accelerator, deploy settings, S3 config | +| `rhaiis.engine` | Active engine: `vllm` (default), `sglang`, or `trtllm` | +| `rhaiis.engines.{vllm,sglang,trtllm}` | Per-engine images, port, and default args | +| `rhaiis.engines.trtllm.trtllm_config` | TRT-LLM server config (KV cache, CUDA graphs, MoE) | +| `rhaiis.deploy` | Deploy settings (replicas, CPU/memory, image_pull_secrets list, storage) | | `rhaiis.s3` | S3 bucket, vault, and credentials for dashboard CSV and profiler trace uploads | | `rhaiis.profiler` | PyTorch profiler settings (enable, S3 prefix, rates, labels) | -| `models` | Model definitions (hf_model_id, per-model vLLM overrides) | +| `models` | Model definitions (hf_model_id, per-model `engine_args` overrides) | | `workloads` | Benchmark profiles (data shape, rates, max_seconds) | -| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret | +| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret, fs_group | | `tests` | CI test mapping (model_key, workload_keys, version) | | `caliper.postprocess` | Caliper postprocessing pipeline (parse, KPI, CSV export) | +## Engine support + +rhaiis supports three inference engines via a generic abstraction: + +| Engine | Config key | Image source | TP arg | Notes | +|--------|-----------|-------------|--------|-------| +| vLLM | `rhaiis.engine: vllm` (default) | `rhaiis.engines.vllm.images` | `tensor-parallel-size` | No root required | +| SGLang | `rhaiis.engine: sglang` | `rhaiis.engines.sglang.images` | `tp-size` | No root required, uses `sglang serve` entrypoint | +| TRT-LLM | `rhaiis.engine: trtllm` | `rhaiis.engines.trtllm.images` | `tp_size` | Requires root (`anyuid` SCC), NGC pull secret | + +Models define `engine_args` with vLLM-style keys (e.g. `tensor-parallel-size`). +When running with SGLang or TRT-LLM, arguments are automatically translated +(e.g. `tensor-parallel-size` → `tp-size` for SGLang, `tp_size` for TRT-LLM). + +Engine-specific args can also be set directly via `rhaiis.engines..args.*` +in FournosJob `configOverrides`, which take precedence over model-level `engine_args`. + +TRT-LLM additionally supports a `trtllm_config` block for server-side configuration +(KV cache, CUDA graphs, MoE backend, etc.) that is serialized as a JSON config file +inside the container. + +KServe manifests (ServingRuntime + InferenceService) are built as Python dicts in +`orchestration/manifests.py` with engine-specific container construction, then +passed to the generic `deploy_kserve_isvc` tool for application. + ## Fournos integration rhaiis supports Fournos-driven execution via `ci.py`: @@ -85,7 +115,7 @@ bin/run_ci rhaiis ci post-cleanup # Detect pipeline failures, cleanu bin/run_ci rhaiis ci export-artifacts # Caliper export to MLflow ``` -### FournosJob YAML +### FournosJob YAML (vLLM) ```yaml apiVersion: fournos.dev/v1 @@ -97,6 +127,12 @@ spec: displayName: rhaiis-benchmark pipeline: forge-full cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications executionEngine: forge: project: rhaiis @@ -106,10 +142,100 @@ spec: rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 tests.rhaiis.version: "vLLM-0.24.0" tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] - rhaiis.vllm_args.tensor-parallel-size: 2 + rhaiis.engines.vllm.args.tensor-parallel-size: 2 rhaiis.cluster_tag: "zeus2" - rhaiis.deploy.image_pull_secret: npalaska-image-pull + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 # opt-in for IBM Cloud clusters + env: + PULL_PULL_SHA: "" +``` + +### FournosJob YAML (SGLang) + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: sglang-benchmark- +spec: + owner: haumesh + displayName: sglang-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "SGLang-0.5.11" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: sglang + rhaiis.engines.sglang.args.tp-size: 2 + rhaiis.engines.sglang.args.disable-radix-cache: true + rhaiis.engines.sglang.args.mem-fraction-static: 0.90 + rhaiis.engines.sglang.args.context-length: 8192 + rhaiis.engines.sglang.args.trust-remote-code: true + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 + env: + PULL_PULL_SHA: "" +``` + +### FournosJob YAML (TRT-LLM) + +TRT-LLM requires the `anyuid` SCC on the target cluster (`oc adm policy add-scc-to-user anyuid -z default -n `). + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: trtllm-benchmark- +spec: + owner: haumesh + displayName: trtllm-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "TRT-LLM-1.3.0rc13" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: trtllm + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull", "ngc-secret"] + rhaiis.deploy.memory_request: "256Gi" + # Engine args + rhaiis.engines.trtllm.args.tp_size: 2 + rhaiis.engines.trtllm.args.ep_size: 2 + rhaiis.engines.trtllm.args.max_batch_size: 256 + rhaiis.engines.trtllm.args.max_num_tokens: 8192 + rhaiis.engines.trtllm.args.trust_remote_code: true + # TRT-LLM server config + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.dtype: fp8 + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.free_gpu_memory_fraction: 0.8 + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.enable_padding: true + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.max_batch_size: 256 + rhaiis.engines.trtllm.trtllm_config.enable_attention_dp: true + rhaiis.engines.trtllm.trtllm_config.moe_config.backend: TRTLLM caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 env: PULL_PULL_SHA: "" ``` @@ -125,7 +251,7 @@ Jobs can also be triggered via PR comments on `openshift-psap/forge`: /var tests.rhaiis.model_key: nemotron3super-120b-fp8 /var tests.rhaiis.version: vLLM-0.24.0 /var tests.rhaiis.workload_keys: ["profile1"] -/var rhaiis.vllm_args.tensor-parallel-size: 2 +/var rhaiis.engines.vllm.args.tensor-parallel-size: 2 ``` Note: `/var` directives use `key: value` format (colon required). @@ -144,18 +270,23 @@ Available configOverrides: | `tests.rhaiis.warmup` | Enable warmup pass before benchmarks | | `tests.rhaiis.slack_user` | Slack user ID for failure notifications | | `tests.rhaiis.compare_version` | Baseline version for regression comparison | +| `rhaiis.engine` | Inference engine: `vllm` (default), `sglang`, `trtllm` | | `rhaiis.namespace` | Kubernetes namespace | | `rhaiis.cluster_tag` | Cluster identifier for dashboard grouping | -| `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `rhaiis.deploy.image_pull_secrets` | List of image pull secret names | | `rhaiis.deploy.storage_pvc` | PVC name for model storage | | `rhaiis.deploy.replicas` | Number of predictor replicas | -| `rhaiis.images.nvidia` | vLLM container image override | -| `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | +| `rhaiis.deploy.memory_request` | Memory request for predictor (e.g. `256Gi` for TRT-LLM) | +| `rhaiis.engines.vllm.args.*` | vLLM CLI args (e.g. `tensor-parallel-size`, `gpu-memory-utilization`) | +| `rhaiis.engines.sglang.args.*` | SGLang CLI args (e.g. `tp-size`, `mem-fraction-static`, `context-length`) | +| `rhaiis.engines.trtllm.args.*` | TRT-LLM CLI args (e.g. `tp_size`, `ep_size`, `max_batch_size`) | +| `rhaiis.engines.trtllm.trtllm_config.*` | TRT-LLM server config (kv_cache, cuda_graph, moe) | | `rhaiis.profiler.enabled` | Enable PyTorch profiler | | `rhaiis.agent_analysis.enabled` | Enable AI agent regression analysis | | `caliper.postprocess.csv_dashboard.enabled` | Enable dashboard CSV S3 sync | | `benchmarks.guidellm.timeout` | Benchmark timeout in seconds | | `benchmarks.guidellm.hf_token_secret` | K8s secret name for HF_TOKEN (omit to skip) | +| `benchmarks.guidellm.fs_group` | Pod-level fsGroup for PVC permissions (disabled by default) | Monitoring Fournos jobs: ```bash @@ -174,6 +305,8 @@ oc patch fournosjob -n psap-automation \ - CI: [`orchestration/ci.py`](./orchestration/ci.py) (Fournos pipeline) - CI test: [`orchestration/test_rhaiis.py`](./orchestration/test_rhaiis.py) - Test phase: [`orchestration/test_phase.py`](./orchestration/test_phase.py) +- Manifests: [`orchestration/manifests.py`](./orchestration/manifests.py) (KServe ServingRuntime + InferenceService builders) +- Runtime config: [`orchestration/runtime_config.py`](./orchestration/runtime_config.py) (engine-aware config helpers) - Analysis: [`orchestration/analysis.py`](./orchestration/analysis.py) - Notifications: [`orchestration/notifications.py`](./orchestration/notifications.py) @@ -181,7 +314,7 @@ oc patch fournosjob -n psap-automation \ | Command | Source | Purpose | |---------|--------|---------| -| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime with configurable labels | +| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Apply pre-built KServe InferenceService + ServingRuntime manifests | | `wait_isvc_ready` | [rhaiis](./toolbox/wait_isvc_ready/) | Poll InferenceService readiness with health check | | `run_guidellm_benchmark` | [canonical](../guidellm/toolbox/run_guidellm_benchmark/) | Run GuideLLM benchmark (shared with llm_d) | | `capture_isvc_state` | [rhaiis](./toolbox/capture_isvc_state/) | Capture InferenceService YAML, pod logs, events, describe output | @@ -253,13 +386,13 @@ Available overrides: |------|---------------|-------------| | `--rates` | `workloads..rates` | Comma-separated concurrency levels (e.g. `1,5,50`) | | `--max-seconds` | `workloads..max_seconds` | Max benchmark duration per rate | -| `--tensor-parallel` | `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | -| `--vllm-image` | `rhaiis.images.` | vLLM container image | +| `--tensor-parallel` | engine args | Tensor parallel size (translated per engine) | +| `--vllm-image` | `rhaiis.engines..images.` | Serving container image | | `--accelerator` | `rhaiis.accelerator` | `nvidia` or `amd` | | `--replicas` | `rhaiis.deploy.replicas` | Number of predictor replicas | | `--storage-source` | `rhaiis.deploy.storage_source` | `hf` (HuggingFace download) or `pvc` | | `--storage-pvc` | `rhaiis.deploy.storage_pvc` | PVC name for model storage | -| `--image-pull-secret` | `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `--image-pull-secret` | `rhaiis.deploy.image_pull_secrets` | Image pull secret name | | `--service-account-name` | `rhaiis.deploy.service_account_name` | Service account for predictor | | `--deployment-name` | derived from model HF ID | InferenceService name | From 618599e1dcd70df80ec5dee289f5a4f0a940fdfe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:02:56 +0200 Subject: [PATCH 0259/1072] [cluster] toolbox: create_sa_kubeconfig: new toolbox script --- .../toolbox/create_sa_kubeconfig/main.py | 349 ++++++++++++++++++ 1 file changed, 349 insertions(+) create mode 100644 projects/cluster/toolbox/create_sa_kubeconfig/main.py diff --git a/projects/cluster/toolbox/create_sa_kubeconfig/main.py b/projects/cluster/toolbox/create_sa_kubeconfig/main.py new file mode 100644 index 000000000..16f1a277e --- /dev/null +++ b/projects/cluster/toolbox/create_sa_kubeconfig/main.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import base64 +import json +import logging +import os +from pathlib import Path + +import yaml + +from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists + +logger = logging.getLogger("DSL") + + +@entrypoint +def run( + sa_name: str, + *, + namespace: str = "", + kubeconfig_file: str, + create_token_secret: bool = True, + token_duration: str = "8760h", +) -> int: + """ + Generate kubeconfig file from a Service Account using non-persisting credential delivery. + + This tool generates a working kubeconfig file that can be used to authenticate as a specific + Service Account. It supports both long-lived token secrets and temporary tokens. + + Args: + sa_name: Name of the Service Account + namespace: Namespace where the Service Account exists/will be created (default: current project) + kubeconfig_file: Output file path for working kubeconfig + create_token_secret: Create a long-lived token secret instead of using kubectl create token + token_duration: Duration for temporary tokens when create_token_secret=False (default: 8760h) + """ + + execute_tasks(locals()) + return 0 + + +@task +def setup_configuration(args, ctx): + """Setup configuration and validate parameters""" + + # Set default namespace from current project if not provided + if args.namespace: + ctx.namespace = args.namespace + else: + result = oc("project", "-q") + ctx.namespace = result.stdout.strip() + if not ctx.namespace: + raise ValueError("Couldn't find the current namespace") + + return f"Prepared to create kubeconfig for SA {args.sa_name} in namespace {ctx.namespace}" + + +@task +def validate_parameters(args, ctx): + """Validate command parameters""" + + if not args.sa_name: + raise ValueError("sa_name cannot be empty") + + # Validate token duration format for temporary tokens + if not args.create_token_secret: + if not args.token_duration.endswith(("s", "m", "h")): + raise ValueError( + "token_duration must end with 's', 'm', or 'h' (e.g., '1h', '30m', '3600s')" + ) + + return f"Validated parameters for Service Account {args.sa_name}" + + +@task +def ensure_service_account(args, ctx): + """Create Service Account if it doesn't exist""" + + if oc_resource_exists("serviceaccount", args.sa_name, namespace=ctx.namespace): + return f"Service Account {args.sa_name} already exists in namespace {ctx.namespace}" + + oc( + "create", + "serviceaccount", + args.sa_name, + "-n", + ctx.namespace, + ) + + return f"Created Service Account {args.sa_name} in namespace {ctx.namespace}" + + +@task +def create_token_secret(args, ctx): + """Create a token secret for the Service Account if requested""" + + if not args.create_token_secret: + return "Using temporary token approach, skipping token secret creation" + + ctx.secret_name = f"{args.sa_name}-token" + + # Check if secret already exists + if oc_resource_exists("secret", ctx.secret_name, namespace=ctx.namespace): + return f"Token secret {ctx.secret_name} already exists in namespace {ctx.namespace}" + + # Create token secret manifest + secret_manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": ctx.secret_name, + "namespace": ctx.namespace, + "annotations": {"kubernetes.io/service-account.name": args.sa_name}, + }, + "type": "kubernetes.io/service-account-token", + } + + # Serialize manifest to YAML and apply through stdin + secret_yaml = yaml.dump(secret_manifest, default_flow_style=False) + oc("apply", "-f", "-", input_text=secret_yaml, handled_secretly=True) + + return f"Created token secret {ctx.secret_name}" + + +@retry(attempts=30, delay=2, backoff=1.0) +@task +def wait_for_token_secret_ready(args, ctx): + """Wait for token secret to be populated""" + + if not args.create_token_secret: + return "Using temporary token approach, no secret to wait for" + + try: + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + return False, f"Secret {ctx.secret_name} not found" + + data = secret_data.get("data", {}) + if not data.get("token") or not data.get("ca.crt"): + return False, f"Secret {ctx.secret_name} not fully populated yet" + + return f"Token secret {ctx.secret_name} is ready" + + except Exception as e: + return False, f"Error checking secret {ctx.secret_name}: {e}" + + +@task +def extract_cluster_info(args, ctx): + """Extract cluster API server URL""" + + # Get API server URL + result = oc("config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}") + ctx.api_server = result.stdout.strip() + + if not ctx.api_server: + raise RuntimeError("Could not extract API server URL from current kubeconfig") + + return f"Extracted cluster info: API server {ctx.api_server}" + + +def _build_cluster_config(ctx): + """Build cluster configuration section for kubeconfig""" + # Use certificate verification by default for security + config = {"server": ctx.api_server} + + # Check if current KUBECONFIG already uses insecure settings (development context) + is_development_context = _is_development_cluster() + + if is_development_context: + config["insecure-skip-tls-verify"] = True + logger.warning("TLS verification disabled - detected development/testing environment") + + return config + + +def _is_development_cluster() -> bool: + """Determine if we're connecting to a development cluster based on current KUBECONFIG.""" + try: + # Get current cluster configuration + result = oc("config", "view", "--minify", "-o", "json", log_stdout=False) + if result.returncode != 0: + return False + + current_config = json.loads(result.stdout) + clusters = current_config.get("clusters", []) + + if not clusters: + return False + + cluster_config = clusters[0].get("cluster", {}) + + if cluster_config.get("insecure-skip-tls-verify"): + logger.debug( + "Development context detected: current cluster uses insecure-skip-tls-verify" + ) + return True + + return False + + except Exception as e: + logger.debug(f"Could not determine cluster context: {e}") + return False + + +@task +def generate_kubeconfig(args, ctx): + """Generate working kubeconfig file using non-persisting credential delivery""" + + # Extract token locally (not stored in context) + if args.create_token_secret: + # Extract token from secret (with secret handling flag to prevent logging) + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + raise RuntimeError(f"Token secret {ctx.secret_name} not found") + + token_b64 = secret_data.get("data", {}).get("token") + if not token_b64: + raise RuntimeError(f"Token not found in secret {ctx.secret_name}") + + token = base64.b64decode(token_b64).decode("utf-8") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + else: + # Create temporary token + result = oc( + "create", + "token", + args.sa_name, + "-n", + ctx.namespace, + f"--duration={args.token_duration}", + handled_secretly=True, + ) + + token = result.stdout.strip() + if not token: + raise RuntimeError("Failed to create temporary token") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + # Create working kubeconfig with token + working_kubeconfig = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "cluster": _build_cluster_config(ctx), + "name": "cluster", + } + ], + "contexts": [ + { + "context": { + "cluster": "cluster", + "namespace": ctx.namespace, + "user": args.sa_name, + }, + "name": args.sa_name, + } + ], + "current-context": args.sa_name, + "users": [{"name": args.sa_name, "user": {"token": token}}], # Working token included + } + + # Write working kubeconfig to specified file path with secure permissions + kubeconfig_path = Path(args.kubeconfig_file) + + with open(kubeconfig_path, "w") as f: + yaml.dump(working_kubeconfig, f, default_flow_style=False) + + # Ensure permissions are correctly set after writing + os.chmod(kubeconfig_path, 0o600) + + # Store path for reference (no tokens or structure stored in context) + ctx.kubeconfig_path = str(kubeconfig_path) + + return f"Generated working kubeconfig: {ctx.kubeconfig_path}" + + +@task +def validate_kubeconfig(args, ctx): + """Validate that the generated kubeconfig works""" + + # Test the kubeconfig by running a simple command + + result = oc( + "--kubeconfig", + ctx.kubeconfig_path, + "whoami", + check=True, + ) + + if result.returncode == 0 and result.stdout.strip(): + username = result.stdout.strip() + return f"Kubeconfig validation successful - authenticated as: {username}" + else: + raise RuntimeError("Authentication test returned no username") + + +@task +def display_usage_instructions(args, ctx): + """Display usage instructions""" + + instructions = f""" +# Working kubeconfig created successfully! + +## Usage: +# Use the kubeconfig file directly +export KUBECONFIG={ctx.kubeconfig_path} +oc get pods + +# Or specify the kubeconfig file explicitly +oc --kubeconfig={ctx.kubeconfig_path} get pods + +## Files created: +- {ctx.kubeconfig_path} (working kubeconfig file with authentication) + +## Security Note: +- Kubeconfig file contains authentication token for the Service Account +- File has restrictive permissions (0600) - only readable by owner +- Uses non-persisting credential delivery (token not stored in execution context) +""" + + if args.create_token_secret: + instructions += f"\n## Token source: Long-lived secret {ctx.secret_name}" + else: + instructions += f"\n## Token source: Temporary token (expires in {args.token_duration})" + + print(instructions) + + return f"Working kubeconfig ready: {ctx.kubeconfig_path}" + + +if __name__ == "__main__": + run.main() From 39fe524325a20b9e09e6f56dfc3ce0358237e85d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 19:15:25 +0200 Subject: [PATCH 0260/1072] [core] dsl: utils: k8s: add the handled secretly arg to flag the calls handling secret data --- projects/core/dsl/utils/k8s.py | 59 ++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 381dd1bed..88381a91a 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -24,31 +24,52 @@ def oc( log_stdout: bool = True, log_stderr: bool = True, stdout_dest: Path | None = None, + handled_secretly: bool = False, ) -> shell.CommandResult: """Run an oc command. Args: *args: Arguments to pass to oc check: Raise subprocess.CalledProcessError if command fails - input_text: Input to send to command (not supported) - timeout_seconds: Command timeout (not supported) + input_text: Input to send to command + timeout_seconds: Command timeout log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) stdout_dest: Optional file path to write stdout to + handled_secretly: If True, disable all logging to prevent secret leakage Returns: CommandResult with execution details """ cmd = ["oc", *args] - return shell.run( - cmd, - check=check, - shell=False, - log_stdout=log_stdout, - log_stderr=log_stderr, - stdout_dest=stdout_dest, - ) + # When handled secretly, disable all logging to prevent credential leakage + if handled_secretly: + log_stdout = False + log_stderr = False + + try: + return shell.run( + cmd, + check=check, + shell=False, + log_stdout=log_stdout, + log_stderr=log_stderr, + stdout_dest=stdout_dest, + input_text=input_text, + timeout_seconds=timeout_seconds, + ) + except subprocess.CalledProcessError as e: + if not handled_secretly: + raise + + # When handled secretly, sanitize error to prevent credential leakage + raise subprocess.CalledProcessError( + e.returncode, + "oc command failed", + "", + "Error output suppressed for secret handling", + ) from e def oc_get_json( @@ -58,6 +79,7 @@ def oc_get_json( namespace: str | None = None, selector: str | None = None, ignore_not_found: bool = False, + handled_secretly: bool = False, ) -> dict[str, Any] | None: """Get a Kubernetes resource as JSON using oc. @@ -67,6 +89,7 @@ def oc_get_json( namespace: Namespace (optional) selector: Label selector (optional) ignore_not_found: Return None instead of raising error if not found + handled_secretly: If True, disable all logging to prevent secret leakage Returns: Resource data as dict, or None if not found and ignore_not_found=True @@ -83,20 +106,30 @@ def oc_get_json( args.extend(["-l", selector]) args.extend(["-o", "json"]) + # When handled secretly, disable all logging to prevent credential leakage + log_stderr_enabled = not ignore_not_found and not handled_secretly + result = oc( *args, check=not ignore_not_found, - log_stdout=False, - log_stderr=not ignore_not_found, + log_stdout=False, # Always disabled to prevent data exposure + log_stderr=log_stderr_enabled, # Disabled when handled secretly + handled_secretly=handled_secretly, ) if result.returncode != 0: if ignore_not_found and _is_oc_not_found_error(result.stderr): return None + # Re-raise error (oc() already sanitized if handled_secretly=True) raise subprocess.CalledProcessError( result.returncode, f"oc {' '.join(args)}", result.stdout, result.stderr ) if not result.stdout: - raise ValueError(f"oc {' '.join(args)} returned no output") + error_msg = ( + f"oc get {kind} returned no output" + if handled_secretly + else f"oc {' '.join(args)} returned no output" + ) + raise ValueError(error_msg) return json.loads(result.stdout) From c5ce243b395197c3c0163d0fcbd699ec20b17eae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 09:47:23 +0200 Subject: [PATCH 0261/1072] AGENTS.md: adjust the secret-handling guidelines --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index b2043d04f..0d2f0c8c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Key paths: If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys), it MUST NOT be written to any path under `env.ARTIFACT_DIR`. -2. **Use `oc_apply_secret()` or `oc("apply", "-f", "-", input_text=...)` for secrets.** +2. **Use `oc("apply", "-f", "-", input_text=..., handled_secretly=True)` for secrets.** Apply secrets via stdin or a dedicated helper that does not persist the payload. Never use the standard `oc_apply()` for Secret-kind manifests. From cec558c97944f243b6ffff202e48d1d8a5b71a19 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 09:53:23 +0200 Subject: [PATCH 0262/1072] [core] dsl: shell: add support for input and timeout --- projects/core/dsl/shell.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/core/dsl/shell.py b/projects/core/dsl/shell.py index fbd833587..8a493e5bd 100644 --- a/projects/core/dsl/shell.py +++ b/projects/core/dsl/shell.py @@ -32,6 +32,8 @@ def run( stdout_dest: str | Path | None = None, log_stdout: bool = True, log_stderr: bool = True, + input_text: str | None = None, + timeout_seconds: float | None = None, ) -> CommandResult: """ Execute a shell command @@ -43,6 +45,8 @@ def run( stdout_dest: Optional file path to write stdout to log_stdout: Optional. If False, don't log the content of stdout. log_stderr: Optional. If False, don't log the content of stderr. + input_text: Optional text to send to command's stdin + timeout_seconds: Optional timeout in seconds Returns: CommandResult with execution details """ @@ -65,6 +69,8 @@ def run( check=False, # We handle check ourselves capture_output=True, text=True, + input=input_text, + timeout=timeout_seconds, ) cmd_result = CommandResult( From 89f906a692da8a7d1f1218398c446bff4e9ef977 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 10:34:35 -0400 Subject: [PATCH 0263/1072] fix: address PR review comments - Add UTF-8 encoding to hierarchical JSON read/write - Validate output paths reject absolute/parent-traversal values - Honor configured output name for hierarchical JSON, save JSONL separately - Fix vLLM config override key in README examples - Mention release tags in FournosJob YAML examples - Accept YAML file paths instead of dicts in deploy_kserve_isvc toolbox - Write manifest files to src/ subdirectory in artifact dir Co-authored-by: Cursor --- projects/caliper/orchestration/postprocess.py | 38 +++++++++++++------ projects/rhaiis/README.md | 8 ++-- projects/rhaiis/orchestration/test_phase.py | 18 ++++++++- .../rhaiis/toolbox/deploy_kserve_isvc/main.py | 32 ++++++---------- 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e94868754..ed60c6af2 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -495,8 +495,13 @@ def _run_artifacts_to_kpis( } try: - # Prepare paths - output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + # Prepare paths — reject absolute or parent-traversal output names + configured_output = postprocess_config.kpi.artifacts_to_kpis.output + if Path(configured_output).is_absolute() or ".." in Path(configured_output).parts: + raise ValueError( + f"kpi.artifacts_to_kpis.output must be a relative path without '..': {configured_output}" + ) + output_file = output_dir / configured_output output_file.parent.mkdir(parents=True, exist_ok=True) # Create temporary status file for subprocess communication @@ -533,7 +538,7 @@ def _run_artifacts_to_kpis( # Read the generated JSONL file kpis = [] - with open(output_file) as f: + with open(output_file, encoding="utf-8") as f: for line in f: line = line.strip() if line: @@ -542,18 +547,20 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Transform to hierarchical format and write to a separate file - # so the original JSONL is preserved for csv-export + # Save the JSONL to a separate file for csv-export, + # then write the hierarchical JSON to the configured output name + jsonl_file = output_file.with_suffix(".jsonl") + output_file.rename(jsonl_file) + hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) import json - hierarchical_file = output_file.with_suffix(".hierarchical.json") - with open(hierarchical_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {hierarchical_file}" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {output_file}" ) else: logger.warning("No KPI records found in output file") @@ -807,8 +814,12 @@ def _run_kpis_to_csv( } try: - # Create output file path - output_file = output_dir / postprocess_config.kpi.kpis_to_csv.output + csv_output = postprocess_config.kpi.kpis_to_csv.output + if Path(csv_output).is_absolute() or ".." in Path(csv_output).parts: + raise ValueError( + f"kpi.kpis_to_csv.output must be a relative path without '..': {csv_output}" + ) + output_file = output_dir / csv_output output_file.parent.mkdir(parents=True, exist_ok=True) # Create temporary status file for subprocess communication @@ -1526,8 +1537,11 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - # Path to the JSON file for reference in command logging - kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + # csv-export reads the JSONL file (not the hierarchical JSON) + configured_output = Path(self.config.kpi.artifacts_to_kpis.output) + kpi_json_path = output_dir / configured_output.with_suffix(".jsonl") + if not kpi_json_path.exists(): + kpi_json_path = output_dir / configured_output result = _run_kpis_to_csv( self.config, plugin, diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index e59c8e29e..5991f39ff 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -139,7 +139,7 @@ spec: args: [nvidia] configOverrides: tests.rhaiis.model_key: nemotron3super-120b-fp8 - rhaiis.images.nvidia: vllm/vllm-openai:v0.24.0 + rhaiis.engines.vllm.images.nvidia: vllm/vllm-openai:v0.24.0 tests.rhaiis.version: "vLLM-0.24.0" tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] rhaiis.engines.vllm.args.tensor-parallel-size: 2 @@ -148,7 +148,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 # opt-in for IBM Cloud clusters env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### FournosJob YAML (SGLang) @@ -187,7 +187,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### FournosJob YAML (TRT-LLM) @@ -237,7 +237,7 @@ spec: caliper.postprocess.csv_dashboard.enabled: true benchmarks.guidellm.fs_group: 0 env: - PULL_PULL_SHA: "" + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` ### GitHub PR workflow diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b0e6c8bb6..246aa4195 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -4,6 +4,9 @@ import logging import os import uuid as _uuid_mod +from pathlib import Path + +import yaml from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels @@ -15,6 +18,12 @@ _warnings: list[str] = [] +def _write_manifest(manifest: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" base = f"{prefix}-{workload_key}-" @@ -225,10 +234,15 @@ def _run_test( service_account_name=deploy_cfg.get("service_account_name", ""), labels=isvc_labels, ) + sr_file = env.ARTIFACT_DIR / "src" / "servingruntime.yaml" + isvc_file = env.ARTIFACT_DIR / "src" / "inferenceservice.yaml" + _write_manifest(sr_manifest, sr_file) + _write_manifest(isvc_manifest, isvc_file) + deploy_kserve_isvc( namespace=namespace, - servingruntime_manifest=sr_manifest, - inferenceservice_manifest=isvc_manifest, + servingruntime_file=str(sr_file), + inferenceservice_file=str(isvc_file), ) logger.info("Waiting for InferenceService to be ready") diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index 4a0f7887e..add66bb25 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - import yaml from projects.core.dsl import ( @@ -18,15 +16,15 @@ def run( *, namespace: str, - servingruntime_manifest: dict, - inferenceservice_manifest: dict, + servingruntime_file: str, + inferenceservice_file: str, ): - """Deploy a KServe InferenceService from pre-built manifests. + """Deploy a KServe InferenceService from pre-rendered YAML files. Args: namespace: Target namespace for the deployment. - servingruntime_manifest: Pre-built ServingRuntime manifest dict. - inferenceservice_manifest: Pre-built InferenceService manifest dict. + servingruntime_file: Path to a ServingRuntime YAML file. + inferenceservice_file: Path to an InferenceService YAML file. """ return execute_tasks(locals()) @@ -41,27 +39,19 @@ def ensure_namespace(args, context): @task def apply_servingruntime(args, context): - output_path = args.artifact_dir / "servingruntime.yaml" - _write_manifest(args.servingruntime_manifest, output_path) - oc("apply", "-f", str(output_path)) - name = args.servingruntime_manifest.get("metadata", {}).get("name", "") + oc("apply", "-f", args.servingruntime_file) + with open(args.servingruntime_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") return f"Applied ServingRuntime {name}" @task def apply_inferenceservice(args, context): - output_path = args.artifact_dir / "inferenceservice.yaml" - _write_manifest(args.inferenceservice_manifest, output_path) - oc("apply", "-f", str(output_path)) - name = args.inferenceservice_manifest.get("metadata", {}).get("name", "") + oc("apply", "-f", args.inferenceservice_file) + with open(args.inferenceservice_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") return f"Applied InferenceService {name}" -def _write_manifest(manifest: dict, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="utf-8") as f: - yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) - - if __name__ == "__main__": run.main() From f5274f7dbdfb857efd1f616dfe348d3bf935e030 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 13:14:47 -0400 Subject: [PATCH 0264/1072] fix: use format.py from PR #136 for csv-export v2 support csv-export now handles both hierarchical JSON (v2) and JSONL formats via read_kpis_from_file, eliminating the need to rename files or maintain separate JSONL copies. The configured output name is honored. Co-authored-by: Cursor --- projects/caliper/cli/commands.py | 14 +- projects/caliper/engine/kpi/format.py | 319 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 13 +- 3 files changed, 325 insertions(+), 21 deletions(-) create mode 100644 projects/caliper/engine/kpi/format.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index a6937755c..4ae1710be 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -544,16 +544,10 @@ def kpi_csv_export( status_data = {"success": False} try: - # Read KPI JSON Lines file (one JSON object per line) - import json - - kpi_records = [] - with open(input_file, encoding="utf-8") as f: - for line in f: - line = line.strip() - if line: # Skip empty lines - kpi_record = json.loads(line) - kpi_records.append(kpi_record) + # Read KPI file (supports both hierarchical JSON and JSONL formats) + from projects.caliper.engine.kpi.format import read_kpis_from_file + + kpi_records = read_kpis_from_file(input_file) # Export to CSV from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py new file mode 100644 index 000000000..3e5eb057d --- /dev/null +++ b/projects/caliper/engine/kpi/format.py @@ -0,0 +1,319 @@ +"""KPI output format transformations and utilities.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: + """ + Transform flat KPI list into hierarchical JSON structure. + + Groups KPIs by test (run_id), extracting common labels and organizing + KPI metadata (name, help, unit, etc.) for improved readability. + + Args: + kpis: List of flat KPI records from compute_kpis + model: Unified model for accessing plugin metadata + + Returns: + Hierarchical JSON structure organized by test + """ + + if not kpis: + return {"schema_version": "2", "tests": []} + + # Group KPIs by test (run_id) + tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) + + # Get KPI function metadata from the plugin module + from projects.caliper.engine.kpi.decorators import get_kpi_functions + + try: + plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) + kpi_functions = get_kpi_functions(plugin_module_obj) + except (ImportError, AttributeError): + kpi_functions = {} + + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + test_data = tests_data[run_id] + + # Extract common labels (excluding KPI-specific ones) + kpi_labels = kpi.get("labels", {}) + test_labels = { + k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] + } # Exclude KPI-specific labels + + # Validate and merge test labels from all KPIs in the test + for key, value in test_labels.items(): + if key in test_data["labels"] and test_data["labels"][key] != value: + raise ValueError( + f"Label mismatch for run_id '{run_id}': key '{key}' has conflicting values " + f"'{test_data['labels'][key]}' vs '{value}'" + ) + test_data["labels"].update(test_labels) + + # Store test metadata from first KPI + if not test_data["metadata"]: + test_data["metadata"] = { + "timestamp": kpi.get("timestamp"), + "source": kpi.get("source", {}), + "run_id": run_id, + } + + # Create KPI record with metadata + kpi_id = kpi.get("kpi_id") + raw_value = kpi.get("value") + + # Resolve _kpi_is_2d first as source of truth when metadata exists + is_2d = False + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + is_2d = getattr(func, "_kpi_is_2d", False) + else: + # Consistent fallback heuristic when metadata is absent + is_2d = bool( + isinstance(raw_value, list) + and raw_value + and isinstance(raw_value[0], (list, tuple)) + and len(raw_value[0]) == 2 + ) + + # Apply tuple-pair structural transform only for confirmed 2D KPIs + if is_2d and isinstance(raw_value, list) and raw_value and len(raw_value) > 0: + first_item = raw_value[0] + if isinstance(first_item, (list, tuple)) and len(first_item) == 2: + try: + # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format + structured_value = { + "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], + "count": len(raw_value), + } + final_value = structured_value + except (ValueError, TypeError, IndexError): + # If conversion fails, preserve original list-of-tuples representation + final_value = raw_value + else: + # Preserve original list-of-tuples representation + final_value = raw_value + else: + # Not 2D or no valid tuple-pair structure, keep original value + final_value = raw_value + + kpi_record = { + "id": kpi_id, + "value": final_value, + "unit": kpi.get("unit"), + "higher_is_better": kpi_labels.get("higher_is_better", True), + "is_2d": is_2d, # Set is_2d consistently based on resolved value + } + + # Add KPI metadata from function decorator if available + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + kpi_record.update( + { + "name": ( + func.__doc__.replace(" KPI.", "") + if func.__doc__ + else kpi_id.replace("_", " ").title() + ), + "help": getattr(func, "_kpi_help", ""), + } + ) + + # Add 2D-specific metadata if this is a 2D KPI + if is_2d: + kpi_record.update( + { + "x_unit": getattr(func, "_kpi_x_unit", ""), + "x_help": getattr(func, "_kpi_x_help", ""), + "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_help": getattr(func, "_kpi_y_help", None) + or getattr(func, "_kpi_help", ""), + } + ) + + # Add formatting info if available + if hasattr(func, "_kpi_format"): + kpi_record["format"] = func._kpi_format + else: + # Fallback if no function metadata available + kpi_record.update( + { + "name": kpi_id.replace("_", " ").title(), + "help": f"KPI: {kpi_id}", + } + ) + + test_data["kpis"].append(kpi_record) + + # Convert to final structure + tests_list = [] + for run_id, test_data in tests_data.items(): + tests_list.append( + { + "run_id": run_id, + "labels": test_data["labels"], + "metadata": test_data["metadata"], + "kpis": test_data["kpis"], + } + ) + + return {"schema_version": "2", "tests": tests_list} + + +def write_kpis_in_format( + kpis: list[dict], output_file: Path, format_type: str = "hierarchical", model: Any = None +) -> None: + """ + Write KPIs to file in the specified format. + + Args: + kpis: List of KPI records + output_file: Path to output file + format_type: Format type - "hierarchical" (default) or "jsonl" + model: Unified model (required for hierarchical format) + """ + output_file.parent.mkdir(parents=True, exist_ok=True) + + if format_type == "hierarchical": + if model is None: + raise ValueError("Model is required for hierarchical format") + + # Transform to hierarchical format (schema v2) + hierarchical_data = transform_kpis_to_hierarchical_format(kpis, model) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) + # Add EOL at EOF if we have data + if kpis: + f.write("\n") + + elif format_type == "jsonl": + # Write as JSONL (schema v1) - matches original behavior + text = "\n".join(json.dumps(kpi, ensure_ascii=False) for kpi in kpis) + ( + "\n" if kpis else "" + ) + with open(output_file, "w", encoding="utf-8") as f: + f.write(text) + + else: + raise ValueError(f"Unknown format type: {format_type}. Use 'hierarchical' or 'jsonl'") + + +def _convert_to_schema_v1_value(raw_value: Any) -> Any: + """ + Convert structured KPI value back to schema-v1 list-of-pairs representation. + + Args: + raw_value: The KPI value, either scalar or structured with data_points/count + + Returns: + Converted value - list of pairs for 2D data, scalar values unchanged + """ + if isinstance(raw_value, dict): + # Check if this is a structured value with data_points + if "data_points" in raw_value and isinstance(raw_value["data_points"], list): + # Convert data_points list back to list-of-pairs format + data_points = raw_value["data_points"] + return [ + [point.get("x"), point.get("y")] + for point in data_points + if isinstance(point, dict) and "x" in point and "y" in point + ] + # For other dict structures, return as-is (preserve existing format) + return raw_value + else: + # Preserve scalar values unchanged + return raw_value + + +def read_kpis_from_file(file_path: Path) -> list[dict]: + """ + Read KPIs from a file, handling both JSONL and hierarchical JSON formats. + + Args: + file_path: Path to the KPI file + + Returns: + List of KPI records in flat format + """ + kpis = [] + + with open(file_path, encoding="utf-8") as f: + content = f.read().strip() + + if not content: + return kpis + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Hierarchical format - flatten it + # Accumulate records in temporary list to ensure atomicity + temp_kpis = [] + + for test in data.get("tests", []): + test_metadata = test.get("metadata", {}) + test_labels = test.get("labels", {}) + + for kpi_record in test.get("kpis", []): + # Convert structured value to schema-v1 format if needed + raw_value = kpi_record.get("value") + converted_value = _convert_to_schema_v1_value(raw_value) + + flat_kpi = { + "schema_version": "1", + "kpi_id": kpi_record.get("id"), + "value": converted_value, + "unit": kpi_record.get("unit"), + "run_id": test_metadata.get("run_id"), + "timestamp": test_metadata.get("timestamp"), + "labels": { + **test_labels, + "higher_is_better": kpi_record.get("higher_is_better", True), + }, + "source": test_metadata.get("source", {}), + } + + # Preserve schema-v2 metadata fields if present + metadata_fields = [ + "is_2d", + "name", + "help", + "x_unit", + "y_unit", + "x_help", + "y_help", + "format", + ] + for field in metadata_fields: + if field in kpi_record: + flat_kpi[field] = kpi_record[field] + temp_kpis.append(flat_kpi) + + # Only merge into main kpis list after entire branch succeeds + kpis.extend(temp_kpis) + else: + # Unknown JSON format + raise ValueError("Unknown JSON format") + + except json.JSONDecodeError: + # Try to parse as JSONL + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + return kpis diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 65c2bae51..71ea9e4ba 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -546,11 +546,6 @@ def _run_artifacts_to_kpis( kpis.append(json.loads(line)) if kpis: - # Save the JSONL to a separate file for csv-export, - # then write the hierarchical JSON to the configured output name - jsonl_file = output_file.with_suffix(".jsonl") - output_file.rename(jsonl_file) - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) import json @@ -559,7 +554,7 @@ def _run_artifacts_to_kpis( json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format: {output_file}" + f"Successfully transformed {len(kpis)} KPI records to hierarchical format" ) else: logger.warning("No KPI records found in output file") @@ -1563,11 +1558,7 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - # csv-export reads the JSONL file (not the hierarchical JSON) - configured_output = Path(self.config.kpi.artifacts_to_kpis.output) - kpi_json_path = output_dir / configured_output.with_suffix(".jsonl") - if not kpi_json_path.exists(): - kpi_json_path = output_dir / configured_output + kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv( self.config, plugin, From e10f7213b69e2a2fe950cb66c25a99f3fed86957 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 13:47:07 -0400 Subject: [PATCH 0265/1072] fix: preserve per-KPI labels (rate_index) in hierarchical JSON Labels that vary across KPIs within the same run_id (e.g. rate_index, intended_concurrency) are now stored on individual KPI records instead of being merged into test-level labels. read_kpis_from_file restores them when flattening, so csv-export produces one row per rate point. Co-authored-by: Cursor --- projects/caliper/engine/kpi/format.py | 54 ++++++++++++------- projects/caliper/orchestration/postprocess.py | 31 +++++++++-- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index 3e5eb057d..7273bb326 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -38,24 +38,33 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: except (ImportError, AttributeError): kpi_functions = {} + # First pass: determine which labels vary across KPIs in the same run + run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + for k, v in kpi.get("labels", {}).items(): + if k == "higher_is_better": + continue + run_label_values[run_id][k].add(str(v)) + + per_kpi_label_keys: dict[str, set[str]] = {} + for run_id, label_vals in run_label_values.items(): + per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} + for kpi in kpis: run_id = kpi.get("run_id", "unknown") test_data = tests_data[run_id] + varying_keys = per_kpi_label_keys.get(run_id, set()) - # Extract common labels (excluding KPI-specific ones) kpi_labels = kpi.get("labels", {}) test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels - - # Validate and merge test labels from all KPIs in the test - for key, value in test_labels.items(): - if key in test_data["labels"] and test_data["labels"][key] != value: - raise ValueError( - f"Label mismatch for run_id '{run_id}': key '{key}' has conflicting values " - f"'{test_data['labels'][key]}' vs '{value}'" - ) - test_data["labels"].update(test_labels) + k: v + for k, v in kpi_labels.items() + if k not in ("higher_is_better",) and k not in varying_keys + } + + if not test_data["labels"]: + test_data["labels"] = test_labels # Store test metadata from first KPI if not test_data["metadata"]: @@ -65,6 +74,9 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: "run_id": run_id, } + # Per-KPI labels that vary across rate points + kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} + # Create KPI record with metadata kpi_id = kpi.get("kpi_id") raw_value = kpi.get("value") @@ -104,13 +116,15 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: # Not 2D or no valid tuple-pair structure, keep original value final_value = raw_value - kpi_record = { + kpi_record: dict[str, Any] = { "id": kpi_id, "value": final_value, "unit": kpi.get("unit"), "higher_is_better": kpi_labels.get("higher_is_better", True), - "is_2d": is_2d, # Set is_2d consistently based on resolved value + "is_2d": is_2d, } + if kpi_specific_labels: + kpi_record["labels"] = kpi_specific_labels # Add KPI metadata from function decorator if available if kpi_id in kpi_functions: @@ -269,6 +283,13 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: raw_value = kpi_record.get("value") converted_value = _convert_to_schema_v1_value(raw_value) + # Merge test-level labels with any per-KPI labels + merged_labels = { + **test_labels, + **kpi_record.get("labels", {}), + "higher_is_better": kpi_record.get("higher_is_better", True), + } + flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), @@ -276,10 +297,7 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: "unit": kpi_record.get("unit"), "run_id": test_metadata.get("run_id"), "timestamp": test_metadata.get("timestamp"), - "labels": { - **test_labels, - "higher_is_better": kpi_record.get("higher_is_better", True), - }, + "labels": merged_labels, "source": test_metadata.get("source", {}), } diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 71ea9e4ba..a32ef2736 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -354,17 +354,33 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: except (ImportError, AttributeError): kpi_functions = {} + # First pass: determine which labels are common (same value for all KPIs in a run) + # vs per-KPI (vary across KPIs, e.g. rate_index, intended_concurrency) + run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + for k, v in kpi.get("labels", {}).items(): + if k == "higher_is_better": + continue + run_label_values[run_id][k].add(str(v)) + + # Labels with more than one distinct value per run are per-KPI + per_kpi_label_keys: dict[str, set[str]] = {} + for run_id, label_vals in run_label_values.items(): + per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} + for kpi in kpis: run_id = kpi.get("run_id", "unknown") test_data = tests_data[run_id] + varying_keys = per_kpi_label_keys.get(run_id, set()) - # Extract common labels (excluding KPI-specific ones) kpi_labels = kpi.get("labels", {}) test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels + k: v + for k, v in kpi_labels.items() + if k not in ("higher_is_better",) and k not in varying_keys + } - # Merge test labels (they should be the same for all KPIs in a test) if not test_data["labels"]: test_data["labels"] = test_labels @@ -376,6 +392,9 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: "run_id": run_id, } + # Per-KPI labels that vary across rate points + kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} + # Create KPI record with metadata kpi_id = kpi.get("kpi_id") raw_value = kpi.get("value") @@ -400,12 +419,14 @@ def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: else: final_value = raw_value - kpi_record = { + kpi_record: dict[str, Any] = { "id": kpi_id, "value": final_value, "unit": kpi.get("unit"), "higher_is_better": kpi_labels.get("higher_is_better", True), } + if kpi_specific_labels: + kpi_record["labels"] = kpi_specific_labels # Add KPI metadata from function decorator if available if kpi_id in kpi_functions: From b95b5c266f3796d083a512921d3079ea0aaad231 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 15:01:20 -0400 Subject: [PATCH 0266/1072] fix: restore run_path when flattening hierarchical KPIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_path was lost during hierarchical→flat conversion, causing all profiles to collapse into the same CSV group key. Now set from run_id. Co-authored-by: Cursor --- projects/caliper/engine/kpi/format.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index 7273bb326..ee7ea1409 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -290,12 +290,14 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: "higher_is_better": kpi_record.get("higher_is_better", True), } + run_id = test_metadata.get("run_id") flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), "value": converted_value, "unit": kpi_record.get("unit"), - "run_id": test_metadata.get("run_id"), + "run_id": run_id, + "run_path": run_id, "timestamp": test_metadata.get("timestamp"), "labels": merged_labels, "source": test_metadata.get("source", {}), From 445a5602809d365211e64a602bccfc3198b4df17 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:14:12 +0200 Subject: [PATCH 0267/1072] [kserve] toolbox: capture_llmisvc_state: capture more information --- .../toolbox/capture_llmisvc_state/main.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index 060963447..c921d45be 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -228,6 +228,62 @@ def capture_llminferenceservice_describe(args, context): return "LLMInferenceService describe captured" +@task +def capture_workload_overview(args, context): + """Capture deployment, replicaset, and pod overview for debugging""" + + workload_overview_path = args.artifact_dir / "artifacts/workload_overview.txt" + + # Capture deployment, replicaset, and pod overview with wide output + shell.run( + f'oc get deploy,rs,pod -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o wide', + stdout_dest=workload_overview_path, + check=False, + ) + + return f"Captured workload overview to {workload_overview_path}" + + +@task +def capture_workload_descriptions(args, context): + """Capture workload descriptions for deployments, replicasets, and pods in a single file""" + + descriptions_file = args.artifact_dir / "artifacts/workload_descriptions.txt" + + with open(descriptions_file, "w") as handle: + # Capture deployments descriptions + handle.write("=== DEPLOYMENTS ===\n") + deploy_result = shell.run( + f'oc describe deployments -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(deploy_result.stdout) + handle.write("\n\n") + + # Capture replicasets descriptions + handle.write("=== REPLICASETS ===\n") + rs_result = shell.run( + f'oc describe replicasets -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(rs_result.stdout) + handle.write("\n\n") + + # Capture pods descriptions + handle.write("=== PODS ===\n") + pods_result = shell.run( + f'oc describe pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(pods_result.stdout) + handle.write("\n") + + return f"Captured workload descriptions to {descriptions_file}" + + @task def capture_pods_describe(args, context): """Capture describe output for related pods""" From 3938c159d7d34a45a8285831986af5638bce4826 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:22:47 +0200 Subject: [PATCH 0268/1072] [kserve] toolbox: deploy_llmisvc: raise an exception if name isn't k8s compliant --- projects/kserve/toolbox/deploy_llmisvc/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index c3efade89..f49f33cf2 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -16,7 +16,7 @@ retry, task, ) -from projects.core.dsl.utils import write_text +from projects.core.dsl.utils import slugify_identifier, write_text from projects.core.dsl.utils.k8s import ( oc, oc_apply, @@ -77,6 +77,17 @@ def copy_manifest_to_src(args, ctx): # Load manifest to extract the service name manifest = load_yaml(original_path) ctx.inference_service_name = manifest["metadata"]["name"] + + # Validate that the service name is Kubernetes compliant + normalized_name = slugify_identifier(ctx.inference_service_name) + if normalized_name != ctx.inference_service_name: + raise ValueError( + f"LLMInferenceService name '{ctx.inference_service_name}' is not Kubernetes compliant. " + f"Expected: '{normalized_name}'. " + f"Names must be lowercase, contain only letters, numbers, and hyphens, " + f"and be 63 characters or less." + ) + ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}" # Ensure the src directory exists From 88ff61b726ecc0c396545b746947695a2c138e78 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 24 Jul 2026 11:11:42 +0200 Subject: [PATCH 0269/1072] [kserve] toolbox: deploy_llmisvc: wait_pods_scheduled: ensure the LLMISVC exists --- .../kserve/toolbox/deploy_llmisvc/main.py | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index f49f33cf2..566ac3660 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -172,10 +172,17 @@ def apply_inference_service(args, ctx): def wait_pods_appear(args, ctx): """Wait for llm-d pods to appear""" - pods = oc_get_json( - "pods", namespace=args.namespace, selector=ctx.selector, ignore_not_found=True + # Use plain text output for better readability in logs + result = oc( + "get", "pods", "-n", args.namespace, "-l", ctx.selector, check=False, log_stdout=True ) - if pods and pods.get("items"): + + # Check if pods appeared (successful command with actual pod output) + if ( + result.returncode == 0 + and result.stdout.strip() + and "No resources found" not in result.stdout + ): return f"Pods appeared for {ctx.inference_service_name}" return False # Retry @@ -254,6 +261,19 @@ def wait_pods_scheduled(args, ctx): service_name = ctx.inference_service_name + # First, check if the LLMInferenceService exists + llmisvc_result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + check=False, + ) + + if llmisvc_result.returncode != 0: + return False, f"LLMInferenceService {service_name} does not exist yet" + # Get pod status using plain text output result = oc( "get", @@ -264,9 +284,16 @@ def wait_pods_scheduled(args, ctx): args.namespace, "--no-headers", check=False, - log_stdout=False, ) + # Check for nonzero return code from pod query + if result.returncode != 0: + return False, f"Failed to query pods for service {service_name}" + + # Check if pod query result indicates no resources found + if "No resources found" in result.stdout: + return False, "No pods found for the service yet" + if not result.stdout.strip(): return False, "No pods found for the service yet" @@ -280,7 +307,7 @@ def wait_pods_scheduled(args, ctx): return f"All pods for {service_name} are scheduled successfully" -@retry(attempts=90, delay=10, backoff=1.0) +@retry(attempts=180, delay=10, backoff=1.0) @task def wait_service_ready(args, ctx): """Wait for LLMInferenceService to be ready""" From 0711987278472618da53b9e5c41df7c75bf851cd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:18:36 +0200 Subject: [PATCH 0270/1072] [guidellm] toolbox: run_guidellm_benchmark: give more time --- projects/guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 3cb443f9d..dc0db6ea8 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -168,7 +168,7 @@ def create_guidellm_resources_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" -@retry(attempts=1080, delay=10, backoff=1.0) +@retry(attempts=360, delay=30, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" From 2de3103bf077f813ea289d7c78d838e397c3ae6b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:25:16 +0200 Subject: [PATCH 0271/1072] [guidellm] toolbox: run_guidellm_benchmark: compress and trim the benchmarks.json file --- .../toolbox/run_guidellm_benchmark/main.py | 181 +++++++++++++----- 1 file changed, 133 insertions(+), 48 deletions(-) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index dc0db6ea8..65cafe544 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -2,6 +2,8 @@ from __future__ import annotations +import gzip +import json import logging import subprocess from pathlib import Path @@ -23,6 +25,23 @@ logger = logging.getLogger(__name__) +def trim_benchmark_json(obj): + """Remove 'requests' field recursively from JSON data""" + if isinstance(obj, dict): + return { + k: trim_benchmark_json(v) + for k, v in obj.items() + if k + not in [ + "requests", + ] + } + elif isinstance(obj, list): + return [trim_benchmark_json(item) for item in obj] + else: + return obj + + @entrypoint def run( *, @@ -35,6 +54,7 @@ def run( guidellm_args: list[str] | None = None, hf_token_secret: str = "", fs_group: int | None = None, + keep_full_benchmark_file: bool = False, ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -49,6 +69,7 @@ def run( guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. fs_group: If set, adds securityContext.fsGroup to the GuideLLM job pod. + keep_full_benchmark_file: Whether to keep the full untrimmed benchmark JSON files alongside trimmed ones (default: False) """ execute_tasks(locals()) @@ -313,7 +334,6 @@ def extract_results(args, ctx): results_dir = args.artifact_dir / "artifacts" / "results" results_dir.mkdir(parents=True, exist_ok=True) - copy_pod = f"{ctx.benchmark_name}-copy" extracted_files: list[dict[str, str | None]] = [] for run in ctx.guidellm_runs: if run.rate is None: @@ -323,10 +343,57 @@ def extract_results(args, ctx): remote_path = f"/results/benchmarks-{run.label}.json" local_path = results_dir / f"benchmarks-{run.label}.json" - local_path = _copy_result_file(ctx.target_namespace, copy_pod, remote_path, local_path) - if local_path is None: + logger.info(f"Retrieving the compressed benchmark file for {run.label}...") + + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") + + result = oc( + "exec", + "-n", + ctx.target_namespace, + f"{ctx.benchmark_name}-copy", + "--", + "gzip", + "-c", + remote_path, + check=False, + log_stdout=False, + stdout_dest=temp_gz_path, + text=False, + ) + if result.returncode != 0: raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") + # Extract and parse JSON from local compressed file + logger.info(f"Extracting compressed benchmark file for {run.label}...") + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + raise RuntimeError( + f"Failed to extract compressed results for run {run.label}: {e}" + ) from e + + # Always create trimmed version + logger.info(f"Trimming the benchmark.json file for {run.label}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON for {run.label}, writing raw data") + write_text(local_path, json_content) + + # Manage full benchmark file based on flag + if not args.keep_full_benchmark_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full benchmark file {temp_gz_path.name}") + else: + logger.info(f"Keeping full benchmark file {temp_gz_path.name}") extracted_files.append( { "label": run.label, @@ -347,69 +414,87 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" -def _copy_result_file(namespace: str, pod: str, remote_path: str, local_path) -> Path | None: - """Copy a result file from a pod, compressing first to handle large files. +def _copy_result_file( + namespace: str, + pod: str, + remote_path: str, + local_path: Path, + *, + trim_json: bool = True, + keep_full_file: bool = False, +) -> Path | None: + """Copy a result file from a pod, with optional compression and trimming. - Falls back to direct ``oc cp`` for small files. + Args: + namespace: Kubernetes namespace + pod: Pod name + remote_path: Path to file inside the pod + local_path: Local destination path + trim_json: Whether to trim large fields from JSON files (default: True) + keep_full_file: Whether to keep the full untrimmed file alongside trimmed one (default: False) + + Returns: + Path to the copied file, or None if copy failed """ import gzip - import shutil - # Try direct oc cp first - result = oc( - "cp", - f"{namespace}/{pod}:{remote_path}", - str(local_path), - "-c", - "copy-helper", - check=False, - log_stdout=False, - ) - if result.returncode == 0 and local_path.exists() and local_path.stat().st_size > 0: - return local_path + # Use compression approach for better handling of large files + logger.info(f"Retrieving compressed file from {remote_path}...") - logging.getLogger(__name__).info( - "Direct oc cp failed (rc=%d), retrying with gzip compression", - result.returncode, - ) + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") - # Compress inside the pod, copy the gz, decompress locally - gz_remote = f"{remote_path}.gz" - compress = oc( + # Compress and stream directly from pod + result = oc( "exec", - pod, "-n", namespace, - "-c", - "copy-helper", + pod, "--", "gzip", - "-kf", + "-c", remote_path, check=False, + log_stdout=False, + stdout_dest=temp_gz_path, + text=False, ) - if compress.returncode != 0: - logging.getLogger(__name__).warning("gzip inside pod failed (rc=%d)", compress.returncode) + if result.returncode != 0: + logging.getLogger(__name__).warning("gzip extraction failed (rc=%d)", result.returncode) return None - local_gz = local_path.parent / f"{local_path.name}.gz" - cp_result = oc( - "cp", - f"{namespace}/{pod}:{gz_remote}", - str(local_gz), - "-c", - "copy-helper", - check=False, - log_stdout=False, - ) - if cp_result.returncode != 0 or not local_gz.exists() or local_gz.stat().st_size == 0: - logging.getLogger(__name__).warning("oc cp of gzipped file failed") - local_gz.unlink(missing_ok=True) + # Extract and parse JSON from local compressed file + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + logging.getLogger(__name__).warning("Failed to extract compressed results: %s", e) + temp_gz_path.unlink(missing_ok=True) return None - with gzip.open(local_gz, "rb") as f_in, open(local_path, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) - local_gz.unlink(missing_ok=True) + # Create trimmed version if requested and file is JSON + if trim_json and local_path.suffix.lower() == ".json": + logger.info(f"Trimming JSON file {local_path.name}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON, writing raw data to {local_path}") + write_text(local_path, json_content) + else: + # Write raw content for non-JSON files or when trimming is disabled + write_text(local_path, json_content) + + # Manage full file based on keep_full_file flag + if not keep_full_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full file {temp_gz_path.name}") + else: + logger.info(f"Keeping full file {temp_gz_path.name}") if local_path.exists() and local_path.stat().st_size > 0: return local_path From 244d836e320aeff9415ef27b9f24095472e66935 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:12:15 +0200 Subject: [PATCH 0272/1072] [guidellm] postprocess: guidellm: improve the plotting --- .../guidellm/plotting/performance_analysis.py | 389 +++++++++++++++--- 1 file changed, 331 insertions(+), 58 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index d1794e2ef..6e8ef1c78 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -637,7 +637,7 @@ def generate_comprehensive_performance_report( report_title: str = "GuideLLM Performance Analysis", ) -> str | None: """ - Generate comprehensive performance analysis report with separate plot files. + Generate comprehensive performance analysis report with separate plots for each loadshape. Args: records: List of unified result records @@ -653,13 +653,21 @@ def generate_comprehensive_performance_report( logger.info(f"\n🚀 Starting {display_title} generation...") logger.info("=" * 70) - df = create_dataframe_from_records(records) - if df.empty: + # Group records by guidellm_loadshape + loadshape_groups = {} + for record in records: + loadshape = record.distinguishing_labels.get("guidellm_loadshape", "default") + if loadshape not in loadshape_groups: + loadshape_groups[loadshape] = [] + loadshape_groups[loadshape].append(record) + + if not loadshape_groups: logger.info("❌ No data available for analysis") return None - logger.info("\n📊 Generating performance analysis plots...") - logger.info(" This may take a moment as we create high-quality visualizations...") + logger.info( + f"\n🔍 Found {len(loadshape_groups)} loadshape(s): {', '.join(loadshape_groups.keys())}" + ) # Generate all the plots as figures plot_functions = [ @@ -680,74 +688,339 @@ def generate_comprehensive_performance_report( report_dir.mkdir(exist_ok=True) logger.info(f"\n📁 Created report directory: {report_dir_name}") - plots_data = [] - for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - logger.info(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") - try: - fig = plot_func(df, title_context) - if fig: - # Save as both PNG and HTML in the report directory - filename = plot_name.lower().replace(" ", "_") - logger.info(f"💾 Saving {plot_name} as PNG and HTML...") - - # Save PNG image - width = 900 if "Percentiles" in plot_name else 800 - height = 600 if "Percentiles" in plot_name else 500 - png_path = save_figure( - fig, report_dir, filename, as_image=True, width=width, height=height - ) - - # Save HTML version - html_path = save_figure(fig, report_dir, filename, as_image=False) + all_plots_data = [] + + # Process each loadshape separately + for loadshape, loadshape_records in loadshape_groups.items(): + logger.info(f"\n📊 Processing loadshape: {loadshape}") + logger.info(f" Records: {len(loadshape_records)}") + + # Create DataFrame for this loadshape + df = create_dataframe_from_records(loadshape_records) + if df.empty: + logger.info(f" ⚠️ No data available for loadshape: {loadshape}") + continue + + logger.info(f" 📈 Generating {len(plot_functions)} plots for {loadshape}...") + + # Create subdirectory for this loadshape + loadshape_dir = report_dir / loadshape + loadshape_dir.mkdir(exist_ok=True) + + # Extract model information for this loadshape + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + loadshape_plots = [] + for i, (plot_name, plot_func) in enumerate(plot_functions, 1): + logger.info( + f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for {loadshape}..." + ) + try: + # Create subtitle with model and loadshape info + subtitle = f"Model: {model_info} • Load Shape: {loadshape}" + + # Add original title context if provided + if title_context: + loadshape_title_context = f"{title_context}
{subtitle}" + else: + loadshape_title_context = f"
{subtitle}" + + fig = plot_func(df, loadshape_title_context) + if fig: + # Save as both PNG and HTML in the loadshape directory + filename = f"{loadshape}_{plot_name.lower().replace(' ', '_')}" + + # Save PNG image + width = 900 if "Percentiles" in plot_name else 800 + height = 600 if "Percentiles" in plot_name else 500 + png_path = save_figure( + fig, loadshape_dir, filename, as_image=True, width=width, height=height + ) - if png_path and html_path: - # Store relative paths for linking - plots_data.append( - ( - plot_name, - f"{report_dir_name}/{Path(png_path).name}", # PNG path - f"{report_dir_name}/{Path(html_path).name}", # HTML path + # Save HTML version + html_path = save_figure(fig, loadshape_dir, filename, as_image=False) + + if png_path and html_path: + # Store relative paths for linking + loadshape_plots.append( + ( + plot_name, + f"{report_dir_name}/{loadshape}/{Path(png_path).name}", # PNG path + f"{report_dir_name}/{loadshape}/{Path(html_path).name}", # HTML path + ) ) + logger.info(f" ✅ {plot_name} saved for {loadshape}") + else: + logger.info( + f" ⚠️ {plot_name} could not be created for {loadshape} (no figure returned)" ) - logger.info(f"✅ {plot_name} saved as PNG and HTML") - else: - logger.info(f"⚠️ {plot_name} could not be created (no figure returned)") - except Exception as e: - logger.info(f"❌ Failed to generate {plot_name}: {e}") + except Exception as e: + logger.info(f" ❌ Failed to generate {plot_name} for {loadshape}: {e}") - logger.info(f"\n✅ Successfully generated {len(plots_data)} visualizations!") + # Store loadshape plots data + if loadshape_plots: + all_plots_data.append((loadshape, loadshape_plots)) - # Generate summary statistics - logger.info("\n📊 Computing performance statistics and insights...") - summary_stats = _generate_performance_summary(df) + if not all_plots_data: + logger.info("❌ No plots were successfully generated") + return None - if summary_stats.get("best_tokens"): - best = summary_stats["best_tokens"] - logger.info(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") + logger.info(f"\n✅ Successfully generated plots for {len(all_plots_data)} loadshape(s)!") - # Create comprehensive HTML report + # Create comprehensive HTML report with sections for each loadshape logger.info("\n📝 Assembling comprehensive HTML report...") - logger.info(" 🔗 Creating report with images linking to interactive HTML versions...") + logger.info(" 🔗 Creating report with loadshape sections and interactive HTML links...") - html_content = _create_comprehensive_html_report_with_images( - plots_data, summary_stats, title_context, display_title - ) + html_content = f""" + + + + + + {display_title} + + + + + +

Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.

+ +
+

{display_title}

+

Generated performance analysis with separate visualizations for each loadshape

+
+ +
+

Quick Navigation:

+
    """ + + # Add navigation links for each loadshape + for loadshape, _ in all_plots_data: + html_content += f'\n
  • {loadshape}
    • ' + + html_content += """ +
+
+""" + + # Add sections for each loadshape with tabbed interface + container_id = 0 + for loadshape, plots in all_plots_data: + # Extract model information for this loadshape + loadshape_records = loadshape_groups[loadshape] + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + html_content += f""" +
+

Loadshape: {loadshape}

+

+ Model: {model_info} • Load Shape: {loadshape} +

+ +
+
""" + + # Define tab mapping with icons + tab_mapping = { + "Token Throughput vs Concurrency": "🚀 Throughput", + "TTFT Analysis": "⏱️ TTFT", + "Token Throughput Percentiles": "📊 Throughput Percentiles", + "Throughput Scaling": "📈 Scaling", + "Latency vs Throughput": "⚖️ Latency Trade-off", + } + + # Create tab headers + for tab_idx, (plot_name, _png_path, _html_path) in enumerate(plots): + tab_title = tab_mapping.get(plot_name, plot_name) + active_class = " active" if tab_idx == 0 else "" + html_content += f""" + """ + + html_content += """ +
""" + + # Create tab contents + for tab_idx, (plot_name, png_path, html_path) in enumerate(plots): + active_class = " active" if tab_idx == 0 else "" + + # Create descriptions for each plot type + descriptions = { + "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", + "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", + "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", + "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", + "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", + } + + description = descriptions.get(plot_name, f"{plot_name} performance analysis.") + + html_content += f""" +
+
+

{plot_name}

+

{description}

+

{plot_name}

+
+
""" + + html_content += """ +
+
""" + container_id += 1 + + html_content += """ + +""" + + # Save the main HTML report + main_html_filename = create_report_filename( + report_title.lower().replace(" ", "_"), report_number, report_title, "html" ) - report_file = output_dir / filename - logger.info(f"\n💾 Writing {display_title} to: {report_file.name}") - report_file.write_text(html_content, encoding="utf-8") + main_html_path = output_dir / main_html_filename + + with open(main_html_path, "w", encoding="utf-8") as f: + f.write(html_content) + + logger.info(f"✅ Comprehensive report saved as: {main_html_filename}") + logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") - file_size_mb = report_file.stat().st_size / (1024 * 1024) - logger.info(f"📄 Report complete! File size: {file_size_mb:.1f}MB") logger.info("=" * 70) - logger.info(f"🎉 {display_title} ready: {report_file.name}") + logger.info(f"🎉 {display_title} ready: {main_html_path.name}") - return str(report_file) + return str(main_html_path) except Exception as e: logger.info(f"❌ Failed to generate comprehensive performance report: {e}") From cd1f5e712707f970ff659ebadf71161134e5b2d1 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:02:11 +0200 Subject: [PATCH 0273/1072] [rhoai] library: deploy: improve the error message --- projects/rhoai/library/deploy.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index f62bd84ba..cefc7b28f 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -79,9 +79,10 @@ def custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: ) if secret_path is None: raise RuntimeError( - "RHOAI pull secret content " - f"'{catalog.pull_secret.vault.content}' was not found in vault " - f"'{catalog.pull_secret.vault.name}'" + f"RHOAI custom catalog pull secret is required but not available. " + f"Vault '{catalog.pull_secret.vault.name}' with content " + f"'{catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." ) return secret_path @@ -286,9 +287,10 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) if pull_secret_path is None: raise RuntimeError( - "RHOAI pull secret content " - f"'{custom_catalog.pull_secret.vault.content}' was not found in vault " - f"'{custom_catalog.pull_secret.vault.name}'" + f"RHOAI custom catalog is enabled, but the required pull secret is not available. " + f"Vault '{custom_catalog.pull_secret.vault.name}' with content " + f"'{custom_catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." ) current_secret = _current_pull_secret_json() @@ -325,9 +327,11 @@ def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: ) if staging_pull_secret_path is None: raise RuntimeError( - "RHOAI staging pull secret content " - f"'{custom_catalog.staging_pull_secret.vault.content}' was not found in vault " - f"'{custom_catalog.staging_pull_secret.vault.name}'" + f"RHOAI custom catalog is enabled with staging pull secret configured, but the " + f"required staging pull secret is not available. " + f"Vault '{custom_catalog.staging_pull_secret.vault.name}' with content " + f"'{custom_catalog.staging_pull_secret.vault.content}' is MANDATORY when staging pull secret is configured. " + f"Please ensure the vault is properly configured and accessible." ) current_secret = _apply_vault_pull_secret( From 5edff8c625264ca4c7423beffe89c96e34c30bda Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:15:10 +0200 Subject: [PATCH 0274/1072] [rhoai] toolbox: wait_datasciencecluster_ready: better detect deployment errors --- .../wait_datasciencecluster_ready/main.py | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py index cd1a7f79f..9053068b0 100644 --- a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py +++ b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py @@ -61,16 +61,14 @@ def capture_initial_dsc(args, ctx): def wait_for_datasciencecluster_ready(args, ctx): """Wait for the DataScienceCluster phase to become Ready""" - # Query only the status.phase field and show output + # Use plain text output to show both READY and REASON columns result = oc( "get", "datasciencecluster", args.datasciencecluster_name, "-n", args.namespace, - "-o", - "jsonpath={.status.phase}", - log_stdout=True, # Show the output + log_stdout=True, # Show the table output check=False, ) @@ -81,18 +79,46 @@ def wait_for_datasciencecluster_ready(args, ctx): error_msg += f": {result.stderr.strip()}" raise RuntimeError(error_msg) - phase = result.stdout.strip() if result.stdout else None + # Parse the table output into a dict + lines = result.stdout.strip().split("\n") + if len(lines) < 2: + parsed_data = {} + else: + header = lines[0].split() + data = lines[1].split() + # Handle case where REASON column might be empty + while len(data) < len(header): + data.append("") + parsed_data = dict(zip(header, data, strict=True)) + + # Check if we have the required columns + if not parsed_data: + return (False, "Unable to parse DataScienceCluster output, waiting...") + + ready_value = parsed_data.get("READY", "").strip() + reason_value = parsed_data.get("REASON", "").strip() + + # Check if READY column shows True + if ready_value.lower() == "true": + return f"DataScienceCluster ready (READY={ready_value})" + + # Fail fast if READY shows False and REASON shows Error + if ready_value.lower() == "false" and "error" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster is in Error state (READY={ready_value}, REASON={reason_value}). Check the DSC status and logs for details." + ) - if phase == "Ready": - return "DataScienceCluster ready" - if phase in {"Failed", "Error"}: - raise RuntimeError(f"DataScienceCluster entered terminal phase {phase}") + # Fail fast if REASON shows Failed + if "failed" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster failed (REASON={reason_value}). Check the DSC status and logs for details." + ) - # Provide specific reason for retry - if not phase: - return (False, "DataScienceCluster status.phase is empty, retrying...") - else: - return (False, f"DataScienceCluster is in {phase} phase, waiting for Ready...") + # Still waiting - provide context for retry + return ( + False, + f"DataScienceCluster is not ready yet (READY={ready_value}, REASON={reason_value or 'N/A'}), waiting...", + ) @always From 875babcad4a0d5b9ca82b410c35746ba05f9c125 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 17:59:45 +0200 Subject: [PATCH 0275/1072] [cluster] toolbox: deploy_custom_catalog: wait for the Pod to start running --- .../toolbox/deploy_custom_catalog/main.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 2b70724f0..113913a54 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -77,6 +77,50 @@ def apply_catalog_source(args, ctx): return f"Applied CatalogSource {args.catalog_source_name}" +@retry(attempts=24, delay=5) # 2 minutes total (24 * 5s) +@task +def wait_for_catalog_source_pod_running(args, ctx): + """Wait for the CatalogSource pod to start running""" + + # Search for pod with the catalog source label + result = shell.run( + f"oc get pods -n {args.catalog_namespace} -l olm.catalogSource={args.catalog_source_name} --no-headers", + check=False, + ) + if not result.success: + return False + + # Check if we have actual pods (not "No resources found") + if "No resources found" in result.stdout: + logger.info(f"No pods found yet for catalogSource {args.catalog_source_name}") + return False + + # Count pods and check for multiple matches + pod_count = len(result.stdout.strip().splitlines()) + if pod_count != 1: + logger.warning( + f"Multiple pods ({pod_count}) found for catalogSource {args.catalog_source_name}" + ) + + # Check for image issues + if "ErrImagePull" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull error: {result.stdout.strip()}") + + if "ImagePullBackOff" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull back off: {result.stdout.strip()}") + + # Check for failed pods first - any failure should cause immediate error + if "Failed" in result.stdout: + raise RuntimeError(f"CatalogSource pod failed to start: {result.stdout.strip()}") + + # Look for Running status in the output + if "Running" in result.stdout: + return "CatalogSource pod is running" + + # Pods exist but not running yet + return False + + @retry(attempts=60, delay=15) @task def wait_for_catalog_source_ready(args, ctx): From f5457e70c571464253e21315457ca925657c14f7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 18:31:18 +0200 Subject: [PATCH 0276/1072] [llm_d] orchestration: render_inference_service: make the deployment k8s compliant --- .../orchestration/render_inference_service.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 29c254333..682021fec 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -73,6 +73,11 @@ def render_inference_service_from_parts( is_pd_deployment = "pd_config" in deployment_profile name = inference_service["name"] + if deployment_profile_name: + name = f"{name}-{deployment_profile_name}" + # Normalize name to be Kubernetes compliant and limit to 25 characters + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name manifest["metadata"]["namespace"] = namespace manifest["metadata"].setdefault("labels", {}) @@ -188,7 +193,10 @@ def _render_standard_deployment( scheduler = deployment_profile.get("scheduler") has_scheduler_manifest = "scheduler_manifest" in deployment_profile - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name manifest["spec"]["replicas"] = deployment_profile["replicas"] @@ -235,7 +243,10 @@ def _render_pd_deployment( from .runtime_config import get_decode_pod_count, get_prefill_pod_count # Set manifest name with deployment profile - manifest["metadata"]["name"] = f"llm-d-{deployment_profile_name}" + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name # Configure prefill section manifest["spec"]["prefill"] = { From 37a512577104833dd8c78f0b93cb95a96457b52b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 18:40:25 +0200 Subject: [PATCH 0277/1072] [llm_d] tests: reference_deployments: normalize the guidellm names --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 2 +- .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 6265d7c82..5be1b330d 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -1,7 +1,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-intelligentrouting-tp4-x4 + name: llm-d-intelligentr-4e3b09 namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index bc538b5ce..ad5475ddb 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -1,7 +1,7 @@ apiVersion: serving.kserve.io/v1alpha1 kind: LLMInferenceService metadata: - name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + name: llm-d-pd-d-x2-p-tp-bfb506 namespace: forge-llm-d annotations: security.opendatahub.io/enable-auth: 'false' From d75dbfe81b06277854c1737cbf2610c88f8f64ef Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 13:49:48 +0200 Subject: [PATCH 0278/1072] [core] dsl: utils: k8s: add the text flag for the oc command --- projects/core/dsl/utils/k8s.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 88381a91a..a843da00c 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -24,6 +24,7 @@ def oc( log_stdout: bool = True, log_stderr: bool = True, stdout_dest: Path | None = None, + text: bool = True, handled_secretly: bool = False, ) -> shell.CommandResult: """Run an oc command. @@ -36,6 +37,7 @@ def oc( log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) stdout_dest: Optional file path to write stdout to + text: Optional. If False, handle binary output (default True) handled_secretly: If True, disable all logging to prevent secret leakage Returns: @@ -58,6 +60,7 @@ def oc( stdout_dest=stdout_dest, input_text=input_text, timeout_seconds=timeout_seconds, + text=text, ) except subprocess.CalledProcessError as e: if not handled_secretly: From f9a05ed1309dadafafffdebfc409f95821cdf4c7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 13:49:29 +0200 Subject: [PATCH 0279/1072] [core] dsl: shell: add text field for the run command --- projects/core/dsl/shell.py | 93 ++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/projects/core/dsl/shell.py b/projects/core/dsl/shell.py index 8a493e5bd..019a0feec 100644 --- a/projects/core/dsl/shell.py +++ b/projects/core/dsl/shell.py @@ -34,6 +34,7 @@ def run( log_stderr: bool = True, input_text: str | None = None, timeout_seconds: float | None = None, + text: bool = True, ) -> CommandResult: """ Execute a shell command @@ -47,6 +48,7 @@ def run( log_stderr: Optional. If False, don't log the content of stderr. input_text: Optional text to send to command's stdin timeout_seconds: Optional timeout in seconds + text: Optional. If False, handle binary output (default True) Returns: CommandResult with execution details """ @@ -68,14 +70,23 @@ def run( shell=shell, check=False, # We handle check ourselves capture_output=True, - text=True, input=input_text, timeout=timeout_seconds, + text=text, ) + # Handle text vs binary output + if text: + stdout_str = result.stdout or "" + stderr_str = result.stderr or "" + else: + # For binary mode, convert to string representation for CommandResult + stdout_str = f"" if result.stdout else "" + stderr_str = result.stderr.decode("utf-8", errors="replace") if result.stderr else "" + cmd_result = CommandResult( - stdout=result.stdout or "", - stderr=result.stderr or "", + stdout=stdout_str, + stderr=stderr_str, returncode=result.returncode, command=command, ) @@ -84,34 +95,60 @@ def run( if stdout_dest: stdout_path = Path(stdout_dest) stdout_path.parent.mkdir(parents=True, exist_ok=True) - with open(stdout_path, "w") as f: - f.write(cmd_result.stdout) + with open(stdout_path, "w" if text else "wb") as f: + f.write(stdout_str if text else result.stdout) # Print output in verbose format - if result.stdout and result.stdout.strip(): - if stdout_dest: - logger.info(f"| ") - elif log_stdout: - stdout = result.stdout.strip().splitlines() - if len(stdout) == 1: - logger.info(f"| {stdout[0]}") + if text: + if result.stdout: + if stdout_dest: + logger.info(f"| ") + elif log_stdout: + stdout_lines = stdout_str.strip().splitlines() + if len(stdout_lines) == 1: + logger.info(f"| {stdout_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stdout_lines) + "\n| ") + else: + logger.info("| ") + + if result.stderr: + if log_stderr: + stderr_lines = stderr_str.strip().splitlines() + if len(stderr_lines) == 1: + logger.info(f"| {stderr_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr_lines) + "\n| ") else: - logger.info("| \n" + "\n| ".join(stdout) + "\n| ") - else: - logger.info("| ") - - if result.stderr and result.stderr.strip(): - if log_stderr: - stderr = result.stderr.strip().splitlines() - if len(stderr) == 1: - logger.info(f"| {stderr[0]}") + logger.info("| ") + + if not (result.stdout or result.stderr): + logger.info("| ") + else: + # Binary mode - always show lengths + if result.stdout: + if stdout_dest: + logger.info( + f"| ({len(result.stdout)} bytes)" + ) else: - logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + logger.info(f"| {len(result.stdout)} bytes") - if not (result.stdout.strip() or result.stderr.strip()): - logger.info("| ") + if result.stderr: + # Try to decode stderr as text for logging (it's usually text even in binary mode) + try: + stderr_text = result.stderr.decode("utf-8", errors="replace").strip() + if stderr_text: + stderr = stderr_text.splitlines() + if len(stderr) == 1: + logger.info(f"| {stderr[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + except Exception: + logger.info(f"| {len(result.stderr)} bytes") + + if not (result.stdout or result.stderr): + logger.info("| ") if result.returncode != 0: logger.info(f"| {result.returncode}") @@ -122,9 +159,9 @@ def run( # Create a more informative error message error_msg = f"Command failed with exit code {result.returncode}: {command}" if result.stderr: - error_msg += f"\nSTDERR: {result.stderr.strip()}" + error_msg += f"\nSTDERR: {stderr_str.strip()}" if result.stdout: - error_msg += f"\nSTDOUT: {result.stdout.strip()}" + error_msg += f"\nSTDOUT: {stdout_str.strip()}" # Create exception with enhanced message error = subprocess.CalledProcessError( From fdb42e08d383f92c567cd6c1007f73bf253bb83d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 19:34:37 +0200 Subject: [PATCH 0280/1072] [llm_d] orchestration: config: disable kpis_to_csv and analyze --- projects/llm_d/orchestration/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index de1e08125..76bc188dc 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -59,7 +59,7 @@ caliper: enabled: true output: kpis/kpis.json kpis_to_csv: - enabled: true + enabled: false output: kpis/kpis.csv include_header_comments: true artifacts_to_ai_data: @@ -67,7 +67,7 @@ caliper: output_dir: ai_data analyze: - enabled: true + enabled: false historical_kpis: historical_data output: regression_analyze/kpi_analyze.json From 81f024d5d414d38d412f57f528071835a2ce6031 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 11:00:45 +0300 Subject: [PATCH 0281/1072] fix: remove invalid kpi.generate and kpi.export fields from config These fields are not recognized by CaliperOrchestrationKpiSection (extra="forbid") and cause pydantic validation to fail during postprocessing. Co-authored-by: Cursor --- projects/mcp_gateway/orchestration/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 38f7c95ef..12f22a939 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -35,12 +35,9 @@ caliper: enabled: false kpi: enabled: true - generate: artifacts_to_kpis: enabled: true output: kpis.json - export: - output: kpis/kpis.json kpis_to_csv: enabled: true output: kpis/kpis.csv From 492ac3364121e9e6d7905c64c7c56a646258e5f6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 14:10:24 +0200 Subject: [PATCH 0282/1072] [fournos_launcher] toolbox: submit_and_wait: Avi's early return --- .../toolbox/submit_and_wait/main.py | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index d0a26d2b1..b9bc9e778 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -216,7 +216,86 @@ def submit_fournos_job(args, ctx): return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" -@retry(attempts=3000, delay=10, backoff=1.0) +@retry(attempts=30, delay=10, backoff=1.0) +@task +def wait_for_job_to_resolve(args, ctx): + """Wait for FOURNOS job to successfully resolve (reach Pending state)""" + + # Check job status + status_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.phase}}"', + check=False, + ) + + if not status_result.success: + # Check if it's a "not found" error (permanent failure) vs temporary error + if "not found" in status_result.stderr.lower(): + raise FournosJobFailureError( + ctx.final_job_name, + "Job not found during resolve phase - may have been deleted or never created", + args.namespace, + "not_found", + ) + + # Other errors might be temporary, retry + logger.info( + f"Failed to get job status during resolve phase, retrying... (stderr: {status_result.stderr.strip()})" + ) + return False # Retry + + status = status_result.stdout.strip() + + if not status: + logger.info(f"Job {ctx.final_job_name} not processed.") + return False, "Not processed" # Retry + + if status == "Resolving": + logger.info(f"Job {ctx.final_job_name} status: {status}. Still resolving, keep waiting.") + return False, "Resolving" # Retry + + if status == "Stopping": + logger.warning(f"Job {ctx.final_job_name} status: {status}.") + raise FournosJobFailureError( + ctx.final_job_name, "Job stopped in its early stages", args.namespace + ) + + if status == "Failed": + # Get failure details + failure_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.message}}"', + check=False, + ) + failure_msg = ( + failure_result.stdout.strip() + if failure_result.success + else "Unknown failure during resolve" + ) + raise FournosJobFailureError( + ctx.final_job_name, f"Job failed in its early stages: {failure_msg}", args.namespace + ) + + if status in ["Admitted", "Running", "Succeeded"]: + # Job moved past Pending - this means resolve succeeded + return f"Job {ctx.final_job_name} resolved successfully (current status: {status})" + + if status == "Pending": + return f"Job {ctx.final_job_name} successfully resolved and reached Pending state" + + logger.info(f"Unknown job {ctx.final_job_name} status: {status}") + return False, "Unknown status" # Unknown status, retry + + +@task +def check_early_return(args, ctx): + """Check for early return""" + + if not args.wait: + return EarlyReturn(f"Fournos job successfully launched: {ctx.final_job_name} (wait=False)") + + return f"Fournos job successfully launched: {ctx.final_job_name}" + + +@retry(attempts=3000, delay=30, backoff=1.0) @task def wait_for_job_completion(args, ctx): """Wait for FOURNOS job to complete""" From 7b7f5509bc70ea09df2476108f1205374e79767e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:02:54 +0200 Subject: [PATCH 0283/1072] docs: caliper: test-labels-format: new doc --- docs/caliper/test-labels-format.md | 130 +++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 docs/caliper/test-labels-format.md diff --git a/docs/caliper/test-labels-format.md b/docs/caliper/test-labels-format.md new file mode 100644 index 000000000..bc2fbec3c --- /dev/null +++ b/docs/caliper/test-labels-format.md @@ -0,0 +1,130 @@ +# Test Labels Format (`__test_labels__.yaml`) + +## Purpose + +The `__test_labels__.yaml` file marks directories as Caliper test bases and provides metadata for test identification, grouping, and status tracking. + +## File Format + +```yaml +version: "1" +labels: + # Test distinguishing characteristics + key1: value1 + key2: value2 +kpi_labels: + # System/environment context for KPI labeling + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + # Test execution status (added at completion) + success: true|false + message: "Status description" +``` + +### Required Fields +- **`version`**: Schema version (currently `"1"`) +- **`labels`**: Key-value pairs describing test characteristics + +### Optional Fields +- **`kpi_labels`**: System/environment context labels for KPI records + - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`) + - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`) + - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`) +- **`completion`**: Test execution status + - **`success`**: `true` if succeeded, `false` if failed + - **`message`**: Human-readable status description + +## Labels vs KPI Labels + +### Labels (Test Configuration) +- **Purpose**: Distinguish different test configurations and variations +- **Usage**: Used for test filtering, grouping, and analysis +- **Examples**: `model_name`, `deployment_profile`, `guidellm_loadshape` +- **Scope**: Specific to the test configuration + +### KPI Labels (System Context) +- **Purpose**: Provide system/environment context for KPI records +- **Usage**: Applied to all KPI metrics from the test, enables cross-environment analysis +- **Examples**: `platform`, `gpu_type`, `test_harness` +- **Scope**: Describes the execution environment + +## Common Labels + +- **`model_name`**: AI/ML model name (e.g., `"llama-3.1-8b"`) +- **`deployment_profile`**: Deployment variant (e.g., `"simple-tp2-x4"`) +- **`guidellm_loadshape`**: Benchmark load pattern (e.g., `"heavy-heterogeneous"`) +- **`benchmark_type`**: Benchmark type (e.g., `"throughput"`, `"latency"`) + +## Common KPI Labels + +- **`platform`**: Target platform (e.g., `"CKS"`, `"RHOAI"`) +- **`gpu_type`**: GPU hardware type (e.g., `"H100"`, `"A100"`) +- **`test_harness`**: Testing framework (e.g., `"guidellm"`, `"vllm"`) + +## Examples + +### Basic Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple" + guidellm_loadshape: "default" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +``` + +### Completed Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple-tp2-x4" + guidellm_loadshape: "heavy-heterogeneous" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + success: true + message: "Test completed successfully" +``` + +### Failed Test +```yaml +version: "1" +labels: + model_name: "gpt-4o" + deployment_profile: "distributed" +kpi_labels: + platform: "RHOAI" + gpu_type: "A100" + test_harness: "vllm" +completion: + success: false + message: "Connection timeout to inference service" +``` + +## Usage + +- **Test Discovery**: Caliper scans for `__test_labels__.yaml` files to find test results +- **Test Analysis**: Labels enable filtering and comparative analysis of test configurations +- **KPI Labeling**: KPI labels are automatically applied to all generated KPI metrics +- **Cross-Environment Analysis**: KPI labels enable performance comparisons across different platforms/hardware +- **Report Generation**: Both labels and KPI labels provide context for charts and report organization +- **Status Tracking**: Completion field tracks test execution outcome + +## Best Practices + +- **Labels**: Use for test-specific configuration and distinguishing characteristics +- **KPI Labels**: Use for system/environment context that applies to all KPIs +- Generate both label sets early, add completion status at test end +- Use consistent, descriptive keys across related tests +- Use lowercase with underscores (e.g., `model_name`, `gpu_type`) +- Keep label schemas stable within projects +- Populate KPI labels from configuration to ensure consistency (e.g., from `cpt.kpi.labels.*`) +- Missing KPI labels should be handled gracefully with fallback values From 6ceec5035e42f1c03f55772267edb2300535c8f5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 11:56:52 +0200 Subject: [PATCH 0284/1072] [llm_d] work on P/D --- projects/llm_d/orchestration/runtime_config.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 19d3ba066..03cd2e7c6 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -161,6 +161,11 @@ def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: match = re.search(r"p\.tp(\d+)", profile_name) if match: return int(match.group(1)) + elif field_name == "decode_tensor_parallelism": + # Look for d.tp4 pattern (decode tensor_parallelism) + match = re.search(r"d\.tp(\d+)", profile_name) + if match: + return int(match.group(1)) else: # Standard patterns: simple-tp4-x4, intelligentrouting-tp4-x4 if field_name == "tensor_parallelism": @@ -200,7 +205,17 @@ def resolve_value(obj: Any, path: str = "") -> Any: return [resolve_value(item, path) for item in obj] elif obj == "FROM_NAME": # Determine which field to extract based on the path - field_name = path.split(".")[-1] # Get the last part of the path + path_parts = path.split(".") + field_name = path_parts[-1] # Get the last part of the path + + # Map field names for P/D deployments + if len(path_parts) >= 2 and path_parts[-2] in ("prefill", "decode"): + container_type = path_parts[-2] # prefill or decode + if field_name == "replicas": + field_name = f"{container_type}_pods" + elif field_name == "tensor_parallelism": + field_name = f"{container_type}_tensor_parallelism" + try: return _extract_value_from_profile_name(profile_name, field_name) except ValueError as e: From 2f2aa7ed561a8bf529934b35a22aea4894da5cbd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:05:34 +0200 Subject: [PATCH 0285/1072] [llm_d] orchestration: add the ability to skip the test finalizers --- projects/llm_d/orchestration/config.d/runtime.yaml | 2 ++ projects/llm_d/orchestration/test_phase.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 81abc4371..05e8739b0 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -15,3 +15,5 @@ router: file: manifests/scheduler_config/__router_template__.yaml placeholder: __ENDPOINT_PICKER_CONFIG__ epp: manifests/scheduler_config + +run_test_finalizers: true diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 8402815d8..93d1d2d7a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -144,7 +144,7 @@ def do_test() -> int: except SignalInterrupt: primary_exc = sys.exc_info() finally: - do_finalizers = True + do_finalizers = config.project.get_config("runtime.run_test_finalizers") if primary_exc and isinstance(primary_exc[1], SignalInterrupt): logging.warning("Caught a SignalInterrupt, skipping the finalizers") do_finalizers = False From 4ac8f62d6173545082cef90b5a088ecb46835041 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 15:07:15 +0200 Subject: [PATCH 0286/1072] [llm_d] orchestration: presets: cks: new preset file --- .../llm_d/orchestration/presets.d/cks.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/cks.yaml diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml new file mode 100644 index 000000000..e96a453c3 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -0,0 +1,58 @@ +__multiple: true + +cks: + runtime.namespace: kpouget-dev + platform.cluster.namespace.labels: {} + + runtime.kserve.wait_readiness: false + runtime.kueue.enabled: true + runtime.kueue.queue_name: llm-d-cluster-queue + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + platform.gateway.status_address_name: gateway-internal + platform.preflight.check_crds: false + runtime.kserve.wait_long_scheduling: true + platform.inference_service.scheduler.node_selector: + gpu.nvidia.com/class: H200 + +cks-intelligent: + extends: + - cks + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-pd: + extends: + - cks + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-smoke: + extends: + - cks + #- llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - intelligentrouting-tp1-x2 + - simple-tp1-x1 + - pd-d.x1-p.tp1-d.tp1-p.x1 + + runtime.benchmark_key: + - short From 0af9e35932b17d9ac74ba84100b0281f3f9b043a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 17:22:54 +0200 Subject: [PATCH 0287/1072] [llm_d] orchestration: allow skipping the kserver readiness wait --- projects/llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 05e8739b0..509634702 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -7,6 +7,7 @@ job_name: null namespace: forge-llm-d kserve: dry_run: false + wait_readiness: true kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 93d1d2d7a..2326bc15a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -180,8 +180,8 @@ def deploy_inference_service() -> str: platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # Check if dry-run mode is enabled early - dry_run = config.project.get_config("runtime.kserve.dry_run", False) + dry_run = config.project.get_config("runtime.kserve.dry_run") + wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: @@ -190,7 +190,7 @@ def deploy_inference_service() -> str: logger.info("Skipping model cache preparation - dry-run mode enabled") # Step 2: Wait for the serving control plane to settle before creating the service. - if not dry_run: + if not dry_run and wait_readiness: rhoai_namespace = platform["rhoai"]["namespace"] wait_kserve_ready.run(namespace=rhoai_namespace) From 5f774852111561de514e39a8c8dc76602c3d074e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 17:39:07 +0200 Subject: [PATCH 0288/1072] [llm_d] improve the kueue handling --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../manifests/kueue-localqueue.yaml | 8 ++++ .../orchestration/render_inference_service.py | 19 +++++++++ projects/llm_d/orchestration/test_phase.py | 42 +++++++++++++++++++ 4 files changed, 70 insertions(+) create mode 100644 projects/llm_d/orchestration/manifests/kueue-localqueue.yaml diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 509634702..d1ad2e850 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -11,6 +11,7 @@ kserve: kueue: enabled: false queue_name: default-queue + local_queue_manifest: manifests/kueue-localqueue.yaml router: template: file: manifests/scheduler_config/__router_template__.yaml diff --git a/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml new file mode 100644 index 000000000..d2232815c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml @@ -0,0 +1,8 @@ +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + name: "QUEUE_NAME" + namespace: "NAMESPACE" +spec: + clusterQueue: "QUEUE_NAME" + stopPolicy: None diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 682021fec..95c64f6c3 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -407,12 +407,19 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: scheduler_template["metadata"] = {} if "annotations" not in scheduler_template["metadata"]: scheduler_template["metadata"]["annotations"] = {} + if "labels" not in scheduler_template["metadata"]: + scheduler_template["metadata"]["labels"] = {} # Apply the same Kueue annotations to the scheduler pod template for annotation_key, annotation_value in kueue_annotations.items(): full_annotation_key = f"{kueue_prefix}{annotation_key}" scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + # Apply the same Kueue labels to the scheduler pod template + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + scheduler_template["metadata"]["labels"][full_label_key] = label_value + # Update the scheduler template back to the data structure manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template @@ -437,3 +444,15 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: manifest["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( pod_group_total_count ) + + # Add required pod-group-name label using the ISVC name + pod_group_name = manifest["metadata"]["name"] + manifest["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name + + # Also add pod-group-name label to scheduler template if it exists + if has_scheduler: + scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + if "metadata" in scheduler_template and "labels" in scheduler_template["metadata"]: + scheduler_template["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = ( + pod_group_name + ) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 2326bc15a..5c40fc1d7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -5,8 +5,11 @@ from pathlib import Path from typing import Any +import yaml + from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier +from projects.core.dsl.utils.k8s import oc from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt @@ -28,6 +31,42 @@ logger = logging.getLogger(__name__) +def ensure_kueue_local_queue() -> None: + """Create LocalQueue when kueue is enabled.""" + enable_kueue = config.project.get_config("runtime.kueue.enabled") + if not enable_kueue: + return + + queue_name = config.project.get_config("runtime.kueue.queue_name") + manifest_path = config.project.get_config("runtime.kueue.local_queue_manifest") + namespace = runtime_config.get_namespace() + + logger.info("Creating LocalQueue: %s", queue_name) + + # Read and parse the YAML template + config_dir = runtime_config.get_config_dir() + template_file = config_dir / manifest_path + + with template_file.open(encoding="utf-8") as f: + local_queue_manifest = yaml.safe_load(f) + + # Update the fields + local_queue_manifest["metadata"]["name"] = queue_name + local_queue_manifest["metadata"]["namespace"] = namespace + local_queue_manifest["spec"]["clusterQueue"] = queue_name + + # Write manifest to manifests directory and apply + manifests_dir = env.ARTIFACT_DIR / "manifests" + manifests_dir.mkdir(parents=True, exist_ok=True) + manifest_file = manifests_dir / f"{queue_name}-localqueue.yaml" + + write_yaml(manifest_file, local_queue_manifest) + + # Apply the manifest + oc("apply", "-f", str(manifest_file)) + logger.info("LocalQueue %s created successfully", queue_name) + + def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" @@ -119,6 +158,9 @@ def do_test() -> int: namespace, labels=config.project.get_config("platform.cluster.namespace.labels") ) + # Ensure LocalQueue exists when kueue is enabled + ensure_kueue_local_queue() + endpoint_url: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None From 4c6237b4eb03b46f13c55cae6117a36b4c81718f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 23:21:50 +0200 Subject: [PATCH 0289/1072] [llm_d] orchestration: correctly cleanup the llmisvc --- projects/llm_d/orchestration/test_phase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 5c40fc1d7..df25e3c61 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -391,9 +391,14 @@ def cleanup_test_resources() -> None: inference_service = platform["inference_service"] smoke = platform["smoke"] + # Build the actual inference service name (includes deployment profile) + base_name = inference_service["name"] + deployment_profile_name = runtime_config.get_deployment_profile_name() + actual_service_name = f"{base_name}-{deployment_profile_name}" + cleanup_test_resources_command.run( namespace=namespace, - inference_service_name=inference_service["name"], + inference_service_name=actual_service_name, smoke_pod_name=smoke["pod_name"], benchmark_job_name=runtime_config.get_benchmark_job_name(), ) From 5a631ea79bbbd33d162c6af6347b2a032011bd33 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 21 Jul 2026 23:22:17 +0200 Subject: [PATCH 0290/1072] [llm_d] toolbox: cleanup_test_resources: delete_inference_service: more accurate message --- projects/llm_d/toolbox/cleanup_test_resources/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 475bf49ec..464c9f420 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -193,7 +193,13 @@ def delete_inference_service(args, ctx): f"in {args.namespace} (rc={result.returncode})" ) - return f"Deleted llminferenceservice {args.inference_service_name}" + # Check if something was actually deleted by looking at the output + if result.stdout.strip(): + return f"Deleted llminferenceservice {args.inference_service_name}" + else: + return ( + f"LLMInferenceService {args.inference_service_name} was not found (already cleaned up)" + ) @retry(attempts=90, delay=10, backoff=1.0) From 03e3f43d1ece410bcba6e84155d42ae0eb5d540b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 12:24:23 +0200 Subject: [PATCH 0291/1072] [llm_d] orchestration: preflight_phase: make more flexible and easier to extend --- .../orchestration/config.d/platform.yaml | 3 + .../llm_d/orchestration/preflight_phase.py | 62 +++++++++++++++---- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 4e666be30..a90b9fd0f 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -95,3 +95,6 @@ smoke: client_image: quay.io/curl/curl:8.21.0 endpoint_path: /v1/completions request_timeout_seconds: 60 + +preflight: + check_crds: true diff --git a/projects/llm_d/orchestration/preflight_phase.py b/projects/llm_d/orchestration/preflight_phase.py index e6c0b17c0..86ab3eddd 100644 --- a/projects/llm_d/orchestration/preflight_phase.py +++ b/projects/llm_d/orchestration/preflight_phase.py @@ -3,22 +3,19 @@ import logging from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.library import ci as ci_lib from projects.llm_d.orchestration import runtime_config logger = logging.getLogger(__name__) -def run() -> int: - """Run preflight checks before testing phase. - - Validates that required CRDs exist in the cluster: - - leaderworkersets.leaderworkerset.x-k8s.io - - llminferenceservices.serving.kserve.io +def check_crds() -> bool: + """Check that required CRDs exist in the cluster. Returns: - 0 on success, non-zero on failure + True if all CRDs are present, False otherwise """ - logger.info("Starting preflight checks") + logger.info("Checking required CRDs") # Get required CRDs from platform configuration platform = runtime_config.get_platform_config() @@ -36,10 +33,49 @@ def run() -> int: logger.info(f"CRD found: {crd_name}") if missing_crds: - logger.error( - f"Preflight check failed - missing {len(missing_crds)} required CRDs: {', '.join(missing_crds)}" + error_message = f"Missing {len(missing_crds)} required CRDs:\n" + "\n".join( + f"- {crd}" for crd in missing_crds ) - return 1 + logger.error(f"CRD check failed - {error_message}") + ci_lib.add_notification_file("MISSING_CRDS", error_message) + return False + + logger.info("CRD check passed - all required CRDs are available") + return True + + +def run() -> int: + """Run preflight checks before testing phase. + + Validates that required components exist in the cluster based on + platform configuration settings. - logger.info("Preflight checks completed successfully - all required CRDs are available") - return 0 + Returns: + 0 on success, non-zero on failure + """ + logger.info("Starting preflight checks") + + platform = runtime_config.get_platform_config() + preflight_config = platform.get("preflight", {}) + + # Track overall success across all checks + all_checks_passed = True + + # Run CRD checks if enabled + if preflight_config.get("check_crds", True): + if not check_crds(): + all_checks_passed = False + else: + logger.info("CRD checks disabled by platform configuration") + + # Future checks can be added here: + # if preflight_config.get("check_operators", True): + # if not check_operators(): + # all_checks_passed = False + + if all_checks_passed: + logger.info("Preflight checks completed successfully") + return 0 + else: + logger.error("One or more preflight checks failed") + return 1 From 6f8ec99f95496c5eafe887567c381be027c40cc3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:17:41 +0200 Subject: [PATCH 0292/1072] [llm_d] orchestration: prepare_phase: make the storage_class_name field mandatory --- projects/llm_d/orchestration/prepare_phase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 533596f69..309b2a714 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -287,7 +287,7 @@ def prepare_model_cache() -> None: "model_uri": model_uri, "pvc_size": model_cache["pvc"]["size"], "access_mode": model_cache["pvc"]["access_mode"], - "storage_class_name": model_cache["pvc"].get("storage_class_name"), + "storage_class_name": model_cache["pvc"]["storage_class_name"], "pvc_name_prefix": pvc_name_prefix, "model_directory_name": model_cache["pvc"]["model_directory_name"], } From 679f735f48fd9f5411cd51a1b24c59de5d43097a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:17:58 +0200 Subject: [PATCH 0293/1072] [llm_d] orchestration: presets: janus: new preset file --- .../llm_d/orchestration/presets.d/janus.yaml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/janus.yaml diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml new file mode 100644 index 000000000..4026bc346 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -0,0 +1,50 @@ +__multiple: true + +janus: + runtime.namespace: kpouget-dev + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + model_cache.pvc.storage_class_name: nfs + deployments.pd.resources: composite.dra.io/gpu-nic-pair + platform.gateway.status_address_name: null + platform.gateway.status_address_name: gateway-internal + +janus-intelligent: + extends: + - janus + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-pd: + extends: + - janus + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-smoke: + extends: + - janus + - llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + + runtime.benchmark_key: + - short From f134f63cace93ff9693c2835e1749f650d165757 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 13:39:51 +0200 Subject: [PATCH 0294/1072] [llm_d] orchestration: test_phase: use MuteStdOut to mute the finalizers --- projects/llm_d/orchestration/test_phase.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index df25e3c61..d091df649 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -109,6 +109,7 @@ def _run_finalizer( **kwargs, ): try: + # with MuteStdOut(reason=f"Finalizer: {description}"): callback(**kwargs) except Exception: if primary_exc is None: @@ -122,24 +123,24 @@ def _run_finalizer( capture_namespace_events = platform["artifacts"]["capture_namespace_events"] finalizer_exc = _run_finalizer( - "capture inference-service state", + "capturing inference-service state", capture_inference_service_state, ) finalizer_exc = _run_finalizer( - "write endpoint URL", + "writing endpoint URL", write_endpoint_url, artifact_dir=env.ARTIFACT_DIR, endpoint_url=endpoint_url, ) finalizer_exc = _run_finalizer( - "capture namespace events", + "capturing namespace events", capture_namespace_events_after_test, artifact_dir=env.ARTIFACT_DIR, namespace=namespace, capture_namespace_events=capture_namespace_events, ) finalizer_exc = _run_finalizer( - "cleanup runtime resources", + "cleaning up runtime resources", cleanup_test_resources, ) From 0aea34d1fc8a35857805d0066b8f814f41a79f05 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 14:50:00 +0200 Subject: [PATCH 0295/1072] [llm_d] orchestration: test_phase: add flag to wait for pod scheduling --- projects/llm_d/orchestration/test_phase.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index d091df649..8271a81f7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -243,11 +243,15 @@ def deploy_inference_service() -> str: # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) + # Get scheduling wait configuration + wait_pods_scheduled = config.project.get_config("runtime.kserve.wait_long_scheduling") + endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), gateway_status_address_name=gateway["status_address_name"], dry_run=dry_run, + wait_pods_scheduled=wait_pods_scheduled, ) if dry_run: From 6cc61128c8f8ee5618b6d6043312b9641e905e55 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 16:29:35 +0200 Subject: [PATCH 0296/1072] [llm_d] orchestration: config: runtime: allow waiting for a long time on kserve readiness --- projects/llm_d/orchestration/config.d/runtime.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index d1ad2e850..90c370119 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -8,6 +8,7 @@ namespace: forge-llm-d kserve: dry_run: false wait_readiness: true + wait_long_scheduling: false kueue: enabled: false queue_name: default-queue From 61bbe24ab19b7c514c3ea6d05335473bee84a5cf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:23:06 +0200 Subject: [PATCH 0297/1072] [llm_d] orchestration: render_inference_service: slugginfy the llmisvc name --- projects/llm_d/orchestration/render_inference_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 95c64f6c3..c893c75b9 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -197,7 +197,6 @@ def _render_standard_deployment( name = slugify_identifier(name) name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name - manifest["spec"]["replicas"] = deployment_profile["replicas"] serving_container = manifest["spec"]["template"]["containers"][0] From 0cd797de64ba9670abfa038ade19fb5f2fa74194 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 22 Jul 2026 21:52:55 +0200 Subject: [PATCH 0298/1072] [llm_d] orchestration: make the scheduler node selector configurable --- .../orchestration/config.d/platform.yaml | 3 +++ .../manifests/llminferenceservice.yaml | 2 -- .../llm_d/orchestration/runtime_config.py | 20 ++++++++++--------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index a90b9fd0f..3f911b06d 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -86,6 +86,9 @@ inference_service: name: llm-d template: manifests/llminferenceservice.yaml workload_deployment_name_suffix: -kserve + scheduler: + node_selector: + nvidia.com/gpu.present: "true" artifacts: capture_namespace_events: true diff --git a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml index c3043bf31..822fcdcbc 100644 --- a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml +++ b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml @@ -17,8 +17,6 @@ spec: template: containers: - name: main - nodeSelector: - nvidia.com/gpu.present: "true" route: {} gateway: {} template: diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 03cd2e7c6..93c071269 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -490,17 +490,19 @@ def get_deployment_profile() -> dict[str, Any]: resolved_profile = deep_merge(resolved_profile, scheduler_content) - # force the scheduler to run on a GPU node. - # 'gpu.present: true' doesn't work on AKS/CKS - scheduler_node_selector = { - "scheduler": { - "template": { - "nodeSelector": {"nvidia.com/gpu.deploy.container-toolkit": "true"}, + # Configure scheduler node selector from platform config + platform_node_selector = config.project.get_config( + "platform.inference_service.scheduler.node_selector" + ) + if platform_node_selector: + scheduler_node_selector = { + "scheduler": { + "template": { + "nodeSelector": platform_node_selector, + } } } - } - - resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) + resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) return resolved_profile From ec20b6a013fce8c0621e6167c114ee0246c61ed9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 09:35:28 +0200 Subject: [PATCH 0299/1072] [llm_d] orchestration: cli: new 'deploy_and_test' entrypoint --- projects/llm_d/orchestration/ci.py | 38 +++++++++--- projects/llm_d/orchestration/cli.py | 71 ++++++++++++++++------ projects/llm_d/orchestration/test_phase.py | 34 +++++++++++ 3 files changed, 118 insertions(+), 25 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index 152603660..de85604bc 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -10,6 +10,7 @@ import click +from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome from projects.core.agentic.config_review import trigger_config_review_for_ci from projects.core.agentic.on_failure import agent_review_on_failure from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint @@ -21,11 +22,11 @@ caliper_export_list_optional_vaults, caliper_export_list_vaults, ) +from projects.core.library.postprocess import run_postprocess_after_test from projects.core.library.replot import caliper_replot_entrypoint from projects.llm_d.orchestration.cleanup_phase import run as cleanup_toolbox_run from projects.llm_d.orchestration.preflight_phase import run as preflight_toolbox_run from projects.llm_d.orchestration.prepare_sequence import run_prepare_sequence -from projects.llm_d.orchestration.test_phase import run as test_toolbox_run logger = logging.getLogger(__name__) RHOAI_CUSTOM_CATALOG_VAULTS = [ @@ -121,17 +122,38 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" - from projects.llm_d.orchestration import runtime_config # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) - exit_code = 0 - for run_spec in runtime_config.get_run_specs(): - with runtime_config.activate_run_spec(run_spec): - with env.NextArtifactDir(run_spec.artifact_dirname): - exit_code = max(exit_code, test_toolbox_run()) - return exit_code + # Run all tests first + from projects.llm_d.orchestration.test_phase import run_all_tests + + max_exit_code = run_all_tests(stop_on_error=False) + + test_failed = max_exit_code != 0 + failure_message = f"Tests completed with exit code {max_exit_code}" if test_failed else None + + # Run post-processing once after all tests + try: + if test_failed and failure_message: + outcome = TestPhaseOutcome("FAILED", failure_message) + elif max_exit_code == 0: + outcome = TestPhaseOutcome("SUCCESS") + else: + outcome = TestPhaseOutcome("FAILED", f"exit_code={max_exit_code}") + + status = run_postprocess_after_test(env.BASE_ARTIFACT_DIR, test_outcome=outcome) + + if status is None or not status.get("success", False): + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + except Exception: + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + return max_exit_code @main.command() diff --git a/projects/llm_d/orchestration/cli.py b/projects/llm_d/orchestration/cli.py index 189b81ecc..362f73791 100755 --- a/projects/llm_d/orchestration/cli.py +++ b/projects/llm_d/orchestration/cli.py @@ -4,7 +4,6 @@ """ import logging -import sys import types from pathlib import Path @@ -16,13 +15,6 @@ logger = logging.getLogger(__name__) -def init(): - """Initialize LLM-D orchestration environment""" - env.init() - run.init() - config.init(Path(__file__).parent) - - @click.group() @click.option( "--preset", @@ -33,18 +25,63 @@ def init(): def main(ctx, preset): """CLI Operations.""" ctx.ensure_object(types.SimpleNamespace) - init() - if not preset: - return + env.init() + run.init() + + if preset: + config.write_variables_override(presets=preset) + + config.init(Path(__file__).parent) + +@main.command() +@click.option( + "--deployment-profile", + multiple=True, + help="The deployment profile(s) to deploy", +) +@click.option( + "--benchmark-key", + multiple=True, + help="The benchmark key(s) to launch.", +) +@click.option( + "--stop-on-error/--continue-on-error", + default=True, + help="Stop on the first test error (default: true)", +) +@click.pass_context +def deploy_and_test(ctx, deployment_profile, benchmark_key, stop_on_error) -> int: + """Test LLM-D with specified deployment profiles and benchmark keys.""" try: - for preset_name in preset: - logger.info(f"Applying preset: {preset_name}") - config.project.apply_preset(preset_name) - except ValueError as e: - logger.error(f"Failed to apply preset '{preset_name}': {e}") - sys.exit(1) + # Configure runtime settings based on provided options + if deployment_profile: + config.project.set_config("runtime.deployment_profile", list(deployment_profile)) + + if benchmark_key: + config.project.set_config("runtime.benchmark_key", list(benchmark_key)) + + # Initialize vaults for deployment + from projects.llm_d.orchestration.ci import init_vaults_for_phase + from projects.llm_d.orchestration.test_phase import run_all_tests + + # Test execution + logger.info("Starting test phase...") + init_vaults_for_phase("test") + + max_exit_code = run_all_tests(stop_on_error=stop_on_error) + + if max_exit_code != 0: + logger.error(f"Test phase completed with exit code {max_exit_code}") + return max_exit_code + + logger.info("All tests completed successfully!") + return 0 + + except Exception: + logger.exception("Deploy and test failed") + return 1 main.add_command(postprocess_command) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 8271a81f7..c5d87a337 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -86,6 +86,40 @@ def create_test_labels() -> None: logger.info("Created test labels: %s", labels) +def run_all_tests(stop_on_error: bool = False) -> int: + """Run tests for all run specifications without post-processing. + + Args: + stop_on_error: If True, stop on the first test failure + + Returns: + Maximum exit code from all tests + """ + from projects.llm_d.orchestration import runtime_config + + max_exit_code = 0 + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + with env.NextArtifactDir(run_spec.artifact_dirname): + try: + exit_code = do_test() + max_exit_code = max(max_exit_code, exit_code) + + if exit_code != 0 and stop_on_error: + logger.error( + f"Test failed with exit code {exit_code}, stopping due to stop_on_error" + ) + return exit_code + except Exception as e: + logger.exception(f"Test failed with exception: {e}") + max_exit_code = 1 + if stop_on_error: + logger.error("Stopping due to stop_on_error") + return 1 + + return max_exit_code + + def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" From 798b762cca60d77eb560d7d5dcb65c175be12365 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:04:11 +0200 Subject: [PATCH 0300/1072] [llm_d] orchestration: improve the P/D config and rendering --- .../orchestration/config.d/deployments.yaml | 13 ++- .../orchestration/render_inference_service.py | 102 ++++++++++++------ .../llm_d/orchestration/runtime_config.py | 16 +++ 3 files changed, 97 insertions(+), 34 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index f25bdfdf8..ac2a14f55 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -40,10 +40,15 @@ profiles: templates: pd: scheduler_manifest: {} # scheduler enabled with default config - tensor_parallelism: FROM_NAME - pd_config: - prefill_pods: FROM_NAME - decode_pods: FROM_NAME + tensor_parallelism: FROM_NAME # Extract main deployment tensor parallelism from profile name + + prefill: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + decode: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + intelligentrouting: scheduler_manifest: {} # scheduler enabled with default config diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index c893c75b9..decfbaf91 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -70,7 +70,7 @@ def render_inference_service_from_parts( manifest = _load_yaml(template_path) # Check if this is a P/D deployment - is_pd_deployment = "pd_config" in deployment_profile + is_pd_deployment = "prefill" in deployment_profile name = inference_service["name"] if deployment_profile_name: @@ -128,6 +128,50 @@ def render_inference_service_from_parts( return rendered_manifest +def handle_pd_resources( + base_resources: dict[str, Any], + deployment_profile: dict[str, Any], + is_prefill: bool = False, +) -> None: + """Handle P/D resource configuration, including DRA support. + + Args: + base_resources: Base resources dict to modify in-place + deployment_profile: Deployment profile configuration + is_prefill: Whether this is for a prefill container + """ + pd_resources = config.project.get_config("deployments.pd.resources") + + if pd_resources == "composite.dra.io/gpu-nic-pair": + # Handle DRA (Dynamic Resource Allocation) case + if is_prefill: + # For prefill pods, use prefill tensor parallelism + tensor_parallelism = deployment_profile["prefill"]["tensor_parallelism"] + else: + # For decode pods, use decode tensor parallelism + tensor_parallelism = deployment_profile["decode"]["tensor_parallelism"] + + # Override GPU resources for DRA + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + # Set nvidia.com/gpu to 0 and add composite DRA resource + base_resources[bound]["nvidia.com/gpu"] = "0" + base_resources[bound][pd_resources] = str(tensor_parallelism) + elif isinstance(pd_resources, dict): + # Normal case: pd_resources is a dictionary to merge + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + base_resources[bound].update(pd_resources) + elif pd_resources is not None: + # Handle unexpected pd_resources type + raise ValueError( + f"Unexpected type for deployments.pd.resources: {type(pd_resources)}, value: {pd_resources}" + ) + # If pd_resources is None, do nothing + + def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: tensor_parallelism = deployment_profile["tensor_parallelism"] profile_resources = deployment_profile.get("resources", {}) @@ -327,13 +371,8 @@ def _build_pd_pod_template( # For decode pods, use main tensor parallelism base_resources = _build_serving_resources(deployment_profile) - # Add P/D extra resources to both requests and limits - pd_resources = config.project.get_config("deployments.pd.resources") - - for bound in ("requests", "limits"): - if bound not in base_resources: - base_resources[bound] = {} - base_resources[bound].update(pd_resources) + # Handle P/D extra resources + handle_pd_resources(base_resources, deployment_profile, is_prefill) # Build container configuration container = { @@ -393,34 +432,29 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: full_annotation_key = f"{kueue_prefix}{annotation_key}" manifest["metadata"]["annotations"][full_annotation_key] = annotation_value - # Apply Kueue annotations to router scheduler pod template if it exists + # Apply Kueue annotations to router scheduler if it exists if ( "spec" in manifest and "router" in manifest["spec"] and "scheduler" in manifest["spec"]["router"] ): - scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) + scheduler = manifest["spec"]["router"]["scheduler"] - # Ensure metadata exists in scheduler template - if "metadata" not in scheduler_template: - scheduler_template["metadata"] = {} - if "annotations" not in scheduler_template["metadata"]: - scheduler_template["metadata"]["annotations"] = {} - if "labels" not in scheduler_template["metadata"]: - scheduler_template["metadata"]["labels"] = {} + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} - # Apply the same Kueue annotations to the scheduler pod template + # Apply the same Kueue annotations to the scheduler for annotation_key, annotation_value in kueue_annotations.items(): full_annotation_key = f"{kueue_prefix}{annotation_key}" - scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value + scheduler["annotations"][full_annotation_key] = annotation_value - # Apply the same Kueue labels to the scheduler pod template + # Apply the same Kueue labels to the scheduler for label_key, label_value in kueue_labels.items(): full_label_key = f"{kueue_prefix}{label_key}" - scheduler_template["metadata"]["labels"][full_label_key] = label_value - - # Update the scheduler template back to the data structure - manifest["spec"]["router"]["scheduler"]["template"] = scheduler_template + scheduler["labels"][full_label_key] = label_value # Calculate pod group total count: 1 scheduler + number of replicas replicas = manifest.get("spec", {}).get("replicas", 1) @@ -448,10 +482,18 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: pod_group_name = manifest["metadata"]["name"] manifest["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name - # Also add pod-group-name label to scheduler template if it exists + # Also add required Kueue annotations/labels to scheduler if it exists if has_scheduler: - scheduler_template = manifest["spec"]["router"]["scheduler"].get("template", {}) - if "metadata" in scheduler_template and "labels" in scheduler_template["metadata"]: - scheduler_template["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = ( - pod_group_name - ) + scheduler = manifest["spec"]["router"]["scheduler"] + + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} + + # Add the same pod-group annotations and labels to scheduler + scheduler["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) + scheduler["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 93c071269..1efebf46c 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -515,12 +515,28 @@ def get_pd_config() -> dict[str, Any]: def get_prefill_pod_count() -> int: """Get number of prefill pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "prefill_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails pd_config = get_pd_config() return pd_config.get("prefill_pods", 1) def get_decode_pod_count() -> int: """Get number of decode pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "decode_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails pd_config = get_pd_config() return pd_config.get("decode_pods", 1) From 4bbdbfa985e3ae4154c3775e2b931401d7e65af7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 11:46:50 +0200 Subject: [PATCH 0301/1072] [llm_d] orchestration: runtime_config: remove legacy check --- projects/llm_d/orchestration/runtime_config.py | 15 ++------------- projects/llm_d/tests/test_profiles.py | 8 -------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 1efebf46c..5dd428446 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -53,16 +53,7 @@ def ensure_artifact_directories(artifact_dir: Path) -> None: def _get_runtime_value(key: str) -> Any: - return config.project.get_config(f"runtime.{key}", None) - - -def _assert_no_legacy_model_key() -> None: - legacy_model_key = _get_runtime_value("model_key") - if legacy_model_key not in (None, ""): - raise ValueError( - "llm_d no longer supports runtime.model_key. " - "Use runtime.model_name with a literal Hugging Face model name instead." - ) + return config.project.get_config(f"runtime.{key}", None, warn=False) def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: @@ -256,7 +247,6 @@ def _load_scheduler_with_epp_config( # Load the router template router_template_path = config_dir / router_template_file if not router_template_path.exists(): - # Fallback to legacy behavior if template doesn't exist raise FileNotFoundError(f"Router template not found at {router_template_path}") # Load the EPP config content @@ -318,7 +308,7 @@ def get_namespace() -> str: def get_model_name() -> str: """Get the selected Hugging Face model name""" - _assert_no_legacy_model_key() + model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") if len(model_names) != 1: raise ValueError( @@ -560,7 +550,6 @@ def get_smoke_request() -> dict[str, Any]: def get_run_specs() -> list[RunSpec]: - _assert_no_legacy_model_key() model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") profile_names = _normalize_string_or_list( _get_runtime_value("deployment_profile"), diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 54180e8dc..6fd10e53c 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -650,14 +650,6 @@ def test_benchmark_deployment_overrides_empty_when_no_benchmark() -> None: assert overrides == {} -def test_runtime_rejects_legacy_model_key_path() -> None: - _init_project_config() - - core_config.project.config.setdefault("runtime", {})["model_key"] = "qwen3-0-6b" - with pytest.raises(ValueError, match="runtime.model_key"): - runtime_config.get_run_specs() - - def test_render_uses_sanitized_model_name_and_profile_resources() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) From b6c0e53713c1318fc8aed0a0551b63843f0bcbc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 12:23:56 +0200 Subject: [PATCH 0302/1072] [llm_d] orchestration: runtime_config: mute some config.project.get_config --- projects/llm_d/orchestration/runtime_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 5dd428446..7b08b0546 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -298,7 +298,9 @@ def get_job_name() -> str: def get_platform_config() -> dict[str, Any]: """Get the normalized platform configuration""" - return normalize_platform_config(copy.deepcopy(config.project.get_config("platform"))) + return normalize_platform_config( + copy.deepcopy(config.project.get_config("platform", print=False)) + ) def get_namespace() -> str: @@ -340,7 +342,7 @@ def get_served_model_name(model_name: str | None = None) -> str: def get_model_cache_config() -> dict[str, Any]: """Get the model cache configuration""" - return copy.deepcopy(config.project.get_config("model_cache")) + return copy.deepcopy(config.project.get_config("model_cache", print=False)) def get_benchmark_keys() -> list[str]: @@ -349,9 +351,9 @@ def get_benchmark_keys() -> list[str]: def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: benchmark = copy.deepcopy( - config.project.get_config(f"workloads.benchmarks['{benchmark_name}']") + config.project.get_config(f"workloads.benchmarks['{benchmark_name}']", print=False) ) - workload_defaults = copy.deepcopy(config.project.get_config("workloads")) + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") for key in default_keys: @@ -438,7 +440,7 @@ def _resolve_template_profile( def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() - deployment_config = copy.deepcopy(config.project.get_config("deployments")) + deployment_config = copy.deepcopy(config.project.get_config("deployments", print=False)) # First try direct profile lookup in profiles section profiles = deployment_config.get("profiles", {}) From fe33de4d866ecc3491e43eeb50b07bceca6999a9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:29:29 +0200 Subject: [PATCH 0303/1072] [llm_d] orchestration: runtime_config: use the deployment in the test dirname --- projects/llm_d/orchestration/runtime_config.py | 2 +- projects/llm_d/tests/test_profiles.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 7b08b0546..257b63502 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -594,7 +594,7 @@ def get_run_specs() -> list[RunSpec]: benchmark_key=bench_key, benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=f"llmd__{bench_key or 'default'}", + artifact_dirname=f"llmd__{bench_key or 'default'}__{profile_slug}", ) ) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 6fd10e53c..9409efce6 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -600,7 +600,7 @@ def test_single_benchmark_key_backward_compatible() -> None: assert len(run_specs) == 1 assert run_specs[0].benchmark_key == "short" assert run_specs[0].benchmark_slug == "short" - assert run_specs[0].artifact_dirname == "llmd__short" + assert run_specs[0].artifact_dirname == "llmd__short__approximate-prefix-cache" def test_activate_run_spec_sets_benchmark_key() -> None: From 50866153b232c7420ed2487b131566af39b2013a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:29:54 +0200 Subject: [PATCH 0304/1072] [llm_d] orchestration: test_phase: don't put the test in the llmd_test dir --- projects/llm_d/orchestration/test_phase.py | 63 +++++++++++----------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c5d87a337..15f8e3173 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -200,39 +200,36 @@ def do_test() -> int: primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None - with env.NextArtifactDir("llmd_test"): - try: - # Create test labels with actual model and profile information - create_test_labels() - - endpoint_url = deploy_inference_service() - - if dry_run: - logging.warning("Running in dry-run mode, skipping the rest of the test steps") - return 0 - - if not endpoint_url: - raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") - run_smoke_request(endpoint_url=endpoint_url) - - run_guidellm_benchmark(endpoint_url=endpoint_url) - except Exception: - primary_exc = sys.exc_info() - except SignalInterrupt: - primary_exc = sys.exc_info() - finally: - do_finalizers = config.project.get_config("runtime.run_test_finalizers") - if primary_exc and isinstance(primary_exc[1], SignalInterrupt): - logging.warning("Caught a SignalInterrupt, skipping the finalizers") - do_finalizers = False - - if dry_run: - do_finalizers = False - - if do_finalizers: - primary_exc, finalizer_exc = run_finalizers( - endpoint_url, primary_exc, finalizer_exc - ) + try: + # Create test labels with actual model and profile information + create_test_labels() + + endpoint_url = deploy_inference_service() + + if dry_run: + logging.warning("Running in dry-run mode, skipping the rest of the test steps") + return 0 + + if not endpoint_url: + raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") + run_smoke_request(endpoint_url=endpoint_url) + + run_guidellm_benchmark(endpoint_url=endpoint_url) + except Exception: + primary_exc = sys.exc_info() + except SignalInterrupt: + primary_exc = sys.exc_info() + finally: + do_finalizers = config.project.get_config("runtime.run_test_finalizers") + if primary_exc and isinstance(primary_exc[1], SignalInterrupt): + logging.warning("Caught a SignalInterrupt, skipping the finalizers") + do_finalizers = False + + if dry_run: + do_finalizers = False + + if do_finalizers: + primary_exc, finalizer_exc = run_finalizers(endpoint_url, primary_exc, finalizer_exc) if primary_exc is not None: raise primary_exc[1].with_traceback(primary_exc[2]) From e361302cd9d9f818a36b976a326ebacdfe2f5d0f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 13:40:33 +0200 Subject: [PATCH 0305/1072] [llm_d] orchestration: test_phase: capture the llmisvc state with the correct name --- projects/llm_d/orchestration/test_phase.py | 58 ++++++++++++++++------ 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 15f8e3173..581cd7621 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -134,6 +134,7 @@ def run() -> int: def run_finalizers( endpoint_url: str | None, + llmisvc_name: str | None, primary_exc: tuple[type[BaseException], BaseException, Any] | None, finalizer_exc: tuple[type[BaseException], BaseException, Any] | None, ) -> tuple[type[BaseException], BaseException, Any] | None: @@ -156,10 +157,16 @@ def _run_finalizer( platform = runtime_config.get_platform_config() capture_namespace_events = platform["artifacts"]["capture_namespace_events"] - finalizer_exc = _run_finalizer( - "capturing inference-service state", - capture_inference_service_state, - ) + # Only capture service state if we have the llmisvc_name + if llmisvc_name: + finalizer_exc = _run_finalizer( + "capturing inference-service state", + capture_inference_service_state, + llmisvc_name=llmisvc_name, + ) + else: + logging.warning("No llmisvc name received, cannot capture the llmisvc state") + finalizer_exc = _run_finalizer( "writing endpoint URL", write_endpoint_url, @@ -197,6 +204,7 @@ def do_test() -> int: ensure_kueue_local_queue() endpoint_url: str | None = None + llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None @@ -204,7 +212,20 @@ def do_test() -> int: # Create test labels with actual model and profile information create_test_labels() - endpoint_url = deploy_inference_service() + # Generate the LLMInferenceService name before deployment + # so we have it available even if deployment fails + from projects.core.dsl.utils import slugify_identifier + + platform = runtime_config.get_platform_config() + inference_service = platform["inference_service"] + base_name = inference_service["name"] + deployment_profile_name = runtime_config.get_deployment_profile_name() + llmisvc_name = ( + f"{base_name}-{deployment_profile_name}" if deployment_profile_name else base_name + ) + llmisvc_name = slugify_identifier(llmisvc_name) + + endpoint_url = deploy_inference_service(llmisvc_name) if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") @@ -229,7 +250,9 @@ def do_test() -> int: do_finalizers = False if do_finalizers: - primary_exc, finalizer_exc = run_finalizers(endpoint_url, primary_exc, finalizer_exc) + primary_exc, finalizer_exc = run_finalizers( + endpoint_url, llmisvc_name, primary_exc, finalizer_exc + ) if primary_exc is not None: raise primary_exc[1].with_traceback(primary_exc[2]) @@ -240,11 +263,14 @@ def do_test() -> int: return 0 -def deploy_inference_service() -> str: - """Deploy LLMInferenceService and return endpoint URL. +def deploy_inference_service(llmisvc_name: str) -> str: + """Deploy LLMInferenceService and return endpoint URL and service name. + + Args: + llmisvc_name: The name of the LLMInferenceService to deploy Returns: - Gateway endpoint URL for the deployed service + Gateway endpoint URL """ logger.info("Starting LLMInferenceService deployment") @@ -253,6 +279,9 @@ def deploy_inference_service() -> str: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] + inference_service = platform["inference_service"] + + # llmisvc_name is now passed as a parameter dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") @@ -289,7 +318,7 @@ def deploy_inference_service() -> str: llmisvc_manifest_path = endpoint_url logger.info("Dry-run completed: LLMInferenceService manifest prepared:") logger.info(llmisvc_manifest_path) - return llmisvc_manifest_path + return llmisvc_manifest_path, llmisvc_name logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url @@ -391,15 +420,12 @@ def run_guidellm_benchmark(*, endpoint_url: str) -> None: ) -def capture_inference_service_state() -> None: - # Load config where it's consumed - +def capture_inference_service_state(llmisvc_name: str) -> None: + """Capture inference service state for the given llmisvc name.""" namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] capture_llmisvc_state.run( - llmisvc_name=inference_service["name"], + llmisvc_name=llmisvc_name, namespace=namespace, ) From e199f0b5f72afd55d350233f789f4c7e0599d1af Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:22:37 +0200 Subject: [PATCH 0306/1072] [llm_d] orchestration: include VLLM args in the workload configuration --- .../orchestration/config.d/deployments.yaml | 1 - .../orchestration/config.d/workloads.yaml | 8 ++++ .../orchestration/render_inference_service.py | 45 +++++++++++++------ .../llm_d/orchestration/runtime_config.py | 34 ++++++++++++++ projects/llm_d/orchestration/test_phase.py | 2 + 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index ac2a14f55..a2b29c769 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -6,7 +6,6 @@ defaults: tensor_parallelism: 1 vllm_extra: args: - - --max-model-len=4096 - --gpu-memory-utilization=0.92 - --trust-remote-code - --no-enable-log-requests diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 5136666c9..0e29ea5eb 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -11,6 +11,10 @@ smoke_requests: temperature: 0.7 benchmarks: + default: + vllm_args: + max_model_len: 4096 + short: timeout_seconds: 900 outputs: json @@ -31,6 +35,8 @@ benchmarks: max_seconds: 600 heavy-heterogeneous: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent @@ -39,6 +45,8 @@ benchmarks: max_seconds: 600 multi-turn: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index decfbaf91..2daee9e10 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -64,6 +64,7 @@ def render_inference_service_from_parts( deployment_profile: dict[str, Any], model_cache: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render an llm_d-owned LLMInferenceService manifest from concrete runtime inputs.""" template_path = Path(config_dir) / inference_service["template"] @@ -115,11 +116,11 @@ def render_inference_service_from_parts( if is_pd_deployment: rendered_manifest = _render_pd_deployment( - manifest, deployment_profile, deployment_profile_name + manifest, deployment_profile, deployment_profile_name, workload ) else: rendered_manifest = _render_standard_deployment( - manifest, deployment_profile, deployment_profile_name + manifest, deployment_profile, deployment_profile_name, workload ) # Apply Kueue configuration if enabled @@ -212,17 +213,25 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: def _build_vllm_additional_args( deployment_profile: dict[str, Any], + workload: dict[str, Any] | None = None, ) -> str: """Build VLLM_ADDITIONAL_ARGS string based on deployment profile configuration. Args: deployment_profile: The deployment profile configuration + workload: The workload configuration (merged benchmark config) Returns: String suitable for VLLM_ADDITIONAL_ARGS environment variable """ - vllm_deploy_args = _build_vllm_args(deployment_profile.get("vllm_extra", {}).get("args")) + vllm_extra = deployment_profile.get("vllm_extra", {}) + vllm_deploy_args = _build_vllm_args(vllm_extra.get("args", {})) + + # Add workload vllm_args if available + if workload and "vllm_args" in workload: + workload_vllm_args = _build_vllm_args(workload["vllm_args"]) + vllm_deploy_args.extend(workload_vllm_args) return " ".join(vllm_deploy_args) @@ -231,6 +240,7 @@ def _render_standard_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render standard (non-P/D) deployment configuration.""" # Check if this is intelligent routing (scheduler_manifest exists) @@ -248,7 +258,7 @@ def _render_standard_deployment( if deployment_profile.get("serving_image"): serving_container["image"] = deployment_profile["serving_image"] - vllm_additional_args = _build_vllm_additional_args(deployment_profile) + vllm_additional_args = _build_vllm_additional_args(deployment_profile, workload) # Add environment variable (don't set generic env vars or args) if "env" not in serving_container: @@ -281,6 +291,7 @@ def _render_pd_deployment( manifest: dict[str, Any], deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render P/D (Prefill/Decode) deployment configuration.""" from .runtime_config import get_decode_pod_count, get_prefill_pod_count @@ -295,14 +306,14 @@ def _render_pd_deployment( manifest["spec"]["prefill"] = { "replicas": get_prefill_pod_count(), "template": _build_pd_pod_template( - deployment_profile, deployment_profile_name, is_prefill=True + deployment_profile, deployment_profile_name, is_prefill=True, workload=workload ), } # Configure main template (decode) manifest["spec"]["replicas"] = get_decode_pod_count() manifest["spec"]["template"] = _build_pd_pod_template( - deployment_profile, deployment_profile_name, is_prefill=False + deployment_profile, deployment_profile_name, is_prefill=False, workload=workload ) return manifest @@ -312,6 +323,7 @@ def _build_pd_pod_template( deployment_profile: dict[str, Any], deployment_profile_name: str | None = None, is_prefill: bool = False, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Build pod template for P/D deployment.""" @@ -325,21 +337,26 @@ def _build_pd_pod_template( # For prefill pods, use prefill tensor parallelism from .runtime_config import _extract_value_from_profile_name - prefill_tp = _extract_value_from_profile_name( - deployment_profile_name, "prefill_tensor_parallelism" - ) - # Create modified profile with prefill tensor parallelism - prefill_profile = copy.deepcopy(deployment_profile) - prefill_profile["tensor_parallelism"] = prefill_tp - base_vllm_args = _build_vllm_additional_args(prefill_profile) + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_vllm_args = _build_vllm_additional_args(prefill_profile, workload) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) else: # For decode pods, use main tensor parallelism - base_vllm_args = _build_vllm_additional_args(deployment_profile) + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) # Add P/D extra args pd_extra_args = pd_vllm_extra.get("args", []) all_vllm_args = base_vllm_args.split() + pd_extra_args + vllm_additional_args = " ".join(all_vllm_args) # Build base environment variables diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index 257b63502..dcfc9a7f1 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -360,6 +360,12 @@ def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: if key in workload_defaults and key not in benchmark: benchmark[key] = workload_defaults[key] + # Merge vllm_args from default benchmark if not present in specific benchmark + if "vllm_args" not in benchmark: + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) + if "vllm_args" in default_benchmark: + benchmark["vllm_args"] = copy.deepcopy(default_benchmark["vllm_args"]) + benchmark_args = benchmark.get("args", {}) workload_args = workload_defaults.get("args", {}) if workload_args: @@ -381,6 +387,34 @@ def get_benchmark_config() -> dict[str, Any] | None: return _resolve_benchmark_config(benchmark_keys[0]) +def get_workload_config() -> dict[str, Any] | None: + """Get workload configuration, falling back to default if no benchmark is specified.""" + benchmark_config = get_benchmark_config() + if benchmark_config: + return benchmark_config + + # No benchmark specified, return default workload configuration + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) + + if not default_benchmark: + return None + + # Apply same merging logic as _resolve_benchmark_config + default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") + for key in default_keys: + if key in workload_defaults and key not in default_benchmark: + default_benchmark[key] = workload_defaults[key] + + # Merge args from workloads.args to default benchmark args + benchmark_args = default_benchmark.get("args", {}) + workload_args = workload_defaults.get("args", {}) + if workload_args: + default_benchmark["args"] = deep_merge(workload_args, benchmark_args) + + return default_benchmark + + def get_benchmark_deployment_overrides() -> dict[str, Any]: """Return deployment overrides from the active benchmark config, if any.""" benchmark = get_benchmark_config() diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 581cd7621..aff9f02da 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -346,6 +346,7 @@ def _build_inference_service_manifest() -> Path: model_slug = runtime_config.get_model_slug(model_name) deployment_profile = runtime_config.get_deployment_profile() model_cache = runtime_config.get_model_cache_config() + workload = runtime_config.get_workload_config() # Get workload config with vllm_args benchmark_overrides = runtime_config.get_benchmark_deployment_overrides() if benchmark_overrides: @@ -362,6 +363,7 @@ def _build_inference_service_manifest() -> Path: deployment_profile=deployment_profile, model_cache=model_cache, deployment_profile_name=deployment_profile_name, + workload=workload, ) # Write the manifest to artifacts From a04fb6918abe4b3b191ce51ea6c5455e73d2459d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:22:55 +0200 Subject: [PATCH 0307/1072] [llm_d] orchestration: presets: new preset for llama3-70b --- .../llm_d/orchestration/presets.d/benchmarks.yaml | 10 ++++++++++ projects/llm_d/orchestration/presets.d/models.yaml | 14 ++++++++++++++ .../llm_d/orchestration/presets.d/presets.yaml | 8 -------- 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 projects/llm_d/orchestration/presets.d/benchmarks.yaml create mode 100644 projects/llm_d/orchestration/presets.d/models.yaml diff --git a/projects/llm_d/orchestration/presets.d/benchmarks.yaml b/projects/llm_d/orchestration/presets.d/benchmarks.yaml new file mode 100644 index 000000000..e9dc9e475 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/benchmarks.yaml @@ -0,0 +1,10 @@ +__multiple: true + +concurrent-1k-1k: + runtime.benchmark_key: concurrent-1k-1k + +heavy-heterogeneous: + runtime.benchmark_key: heavy-heterogeneous + +multi-turn: + runtime.benchmark_key: multi-turn diff --git a/projects/llm_d/orchestration/presets.d/models.yaml b/projects/llm_d/orchestration/presets.d/models.yaml new file mode 100644 index 000000000..995860310 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/models.yaml @@ -0,0 +1,14 @@ +__multiple: true + +gpt-oss: + runtime.model_name: openai/gpt-oss-120b + runtime.deployment_profile: distributed-default + runtime.smoke_request_key: default + model_cache.pvc.size: 300Gi + workloads.args: + request_type: text_completions + +llama3-70b: + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic + model_cache.pvc.size: 100Gi + runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 491296ed8..6b42bb701 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -38,14 +38,6 @@ benchmark-short: runtime.smoke_request_key: default runtime.benchmark_key: short -gpt-oss: - runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: distributed-default - runtime.smoke_request_key: default - model_cache.pvc.size: 300Gi - workloads.args: - request_type: text_completions - gpt-oss-120b-inference-scheduling-release: extends: - gpt-oss From 0655e23368a6f039f0715bb7049cfd0a0582bdde Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 17:23:18 +0200 Subject: [PATCH 0308/1072] [llm_d] tests: test_deployment_profiles: disable the post-processing --- projects/llm_d/tests/test_deployment_profiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 0663760a3..f23a2731e 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -53,6 +53,7 @@ def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): yaml.safe_dump( { "runtime.kserve.dry_run": True, + "caliper.postprocess.enabled": False, }, sort_keys=True, ), @@ -229,7 +230,7 @@ def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: def _find_generated_llmisvc(artifact_dir: Path) -> Path: - """Find the generated LLMISVC manifest in the artifact directory (legacy single deployment).""" + """Find the generated LLMISVC manifest in the artifact directory.""" deployments = _find_generated_llmisvcs(artifact_dir) if len(deployments) == 1: From 381ee4e4933d1278b4b3df8b5e00903c319e2185 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:10:03 +0200 Subject: [PATCH 0309/1072] [llm_d] orchestration: add a config flag to reuse the existing llmisvc --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 48 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 90c370119..e306b1a01 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -9,6 +9,7 @@ kserve: dry_run: false wait_readiness: true wait_long_scheduling: false + reuse_existing: false kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index aff9f02da..f99588dd6 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -279,12 +279,58 @@ def deploy_inference_service(llmisvc_name: str) -> str: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - inference_service = platform["inference_service"] # llmisvc_name is now passed as a parameter dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + # Check if we should reuse existing LLMInferenceService + if reuse_existing and not dry_run: + logger.info(f"Checking if LLMInferenceService {llmisvc_name} already exists") + + # Check if the service already exists + try: + existing_llmisvc = oc( + "get", + "llminferenceservice", + llmisvc_name, + "-n", + namespace, + check=False, + ) + + if existing_llmisvc.returncode == 0: + logger.info( + f"Found existing LLMInferenceService {llmisvc_name}, attempting to reuse" + ) + + # Import and use the toolbox function to extract URL + from projects.kserve.toolbox.deploy_llmisvc import try_resolve_endpoint_url + + endpoint_url = try_resolve_endpoint_url( + namespace=namespace, + inference_service_name=llmisvc_name, + gateway_status_address_name=gateway["status_address_name"], + ) + + if endpoint_url: + logger.info(f"Successfully reused existing LLMInferenceService: {endpoint_url}") + return endpoint_url + else: + logger.warning( + "Existing LLMInferenceService found but no endpoint URL could be resolved, proceeding with new deployment" + ) + else: + logger.info( + f"LLMInferenceService {llmisvc_name} does not exist, proceeding with new deployment" + ) + + except Exception as e: + logger.warning( + f"Error checking for existing LLMInferenceService: {e}, proceeding with new deployment" + ) # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: From c6a27ea6d338d6da167e2a38927db10444e7dadd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 18:49:26 +0200 Subject: [PATCH 0310/1072] [llm_d] orchestration: test_phase: delete the llmisvc & workloads on start --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index e306b1a01..996064747 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -10,6 +10,7 @@ kserve: wait_readiness: true wait_long_scheduling: false reuse_existing: false + delete_all_on_start: true kueue: enabled: false queue_name: default-queue diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f99588dd6..113665f5f 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -31,6 +31,77 @@ logger = logging.getLogger(__name__) +def _delete_resources_by_type(resource_type: str, namespace: str, description: str) -> None: + """Delete all resources of a given type in the namespace. + + Args: + resource_type: Kubernetes resource type (e.g., 'llminferenceservice', 'workload') + namespace: Target namespace + description: Human-readable description for logging + """ + logger.info("Deleting all %s in namespace %s", description, namespace) + + result = oc( + "get", + resource_type, + "-n", + namespace, + "--no-headers", + "-o", + "name", + check=False, + ) + + if result.returncode == 0 and result.stdout.strip(): + resource_names = result.stdout.strip().split("\n") + logger.info( + "Found %d %s to delete: %s", + len(resource_names), + description, + ", ".join(resource_names), + ) + + # Delete all found resources + for resource_name in resource_names: + logger.info("Deleting %s", resource_name) + oc("delete", resource_name, "-n", namespace, check=False) + + logger.info("Successfully deleted all existing %s", description) + else: + logger.info("No existing %s found in namespace %s", description, namespace) + + +def cleanup_existing_resources(namespace: str) -> None: + """Delete all existing LLMInferenceServices and Kueue workloads if configured. + + Args: + namespace: Target namespace for cleanup + """ + delete_all_on_start = config.project.get_config("runtime.kserve.delete_all_on_start", False) + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + if not delete_all_on_start: + return + + if reuse_existing: + logger.info("Skipping cleanup - reuse_existing is enabled") + return + + logger.info("Starting cleanup of existing resources in namespace %s", namespace) + + try: + # Delete LLMInferenceServices + _delete_resources_by_type("llminferenceservice", namespace, "LLMInferenceServices") + + # Also delete Kueue workloads if Kueue is enabled + enable_kueue = config.project.get_config("runtime.kueue.enabled", False) + if enable_kueue: + _delete_resources_by_type("workload", namespace, "Kueue workloads") + + except Exception as e: + logger.warning("Failed to delete existing resources: %s, continuing with test", e) + + def ensure_kueue_local_queue() -> None: """Create LocalQueue when kueue is enabled.""" enable_kueue = config.project.get_config("runtime.kueue.enabled") @@ -203,6 +274,9 @@ def do_test() -> int: # Ensure LocalQueue exists when kueue is enabled ensure_kueue_local_queue() + # Delete all existing resources if configured + cleanup_existing_resources(namespace) + endpoint_url: str | None = None llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None From 89ff380aee8a9fd50e69bb6fb04c7221bf59be57 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 19:09:29 +0200 Subject: [PATCH 0311/1072] [llm_d] tests: reference_deployments: update --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 6 +++--- .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 10 ++++------ .../deployment-simple-tp4-x4/llmisvc.yaml | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index 5be1b330d..c4a16a88b 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -22,7 +22,7 @@ spec: containers: - name: main nodeSelector: - nvidia.com/gpu.deploy.container-toolkit: 'true' + nvidia.com/gpu.present: 'true' route: {} gateway: {} template: @@ -58,5 +58,5 @@ spec: timeoutSeconds: 30 env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index ad5475ddb..8dbe6a352 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -21,8 +21,6 @@ spec: template: containers: - name: main - nodeSelector: - nvidia.com/gpu.present: 'true' route: {} gateway: {} template: @@ -37,8 +35,8 @@ spec: rdma/ib: '1' env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - name: VLLM_NIXL_SIDE_CHANNEL_HOST valueFrom: @@ -59,8 +57,8 @@ spec: rdma/ib: '1' env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching --block-size 128 --kv-transfer-config + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - name: VLLM_NIXL_SIDE_CHANNEL_HOST valueFrom: diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index 6e3edb0be..f17b89144 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -49,5 +49,5 @@ spec: timeoutSeconds: 30 env: - name: VLLM_ADDITIONAL_ARGS - value: --max-model-len=4096 --gpu-memory-utilization=0.92 --trust-remote-code - --no-enable-log-requests --enable-prefix-caching + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 From e2903884cb27da8a04614e8cc9ce856512928c27 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:33:34 +0200 Subject: [PATCH 0312/1072] [llm_d] orchestration: test_phase: cleanup with the right llmisvc name --- projects/llm_d/orchestration/test_phase.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 113665f5f..c50d9958c 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -251,9 +251,11 @@ def _run_finalizer( namespace=namespace, capture_namespace_events=capture_namespace_events, ) + finalizer_exc = _run_finalizer( "cleaning up runtime resources", cleanup_test_resources, + llmisvc_name=llmisvc_name or "llmisvc-name-not-available", ) return primary_exc, finalizer_exc @@ -561,8 +563,12 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: endpoint_file.write_text(f"{endpoint_url}\n", encoding="utf-8") -def cleanup_test_resources() -> None: - """Cleanup test resources using the toolbox script""" +def cleanup_test_resources(llmisvc_name: str | None) -> None: + """Cleanup test resources using the toolbox script + + Args: + llmisvc_name: The actual LLMInferenceService name that was deployed, or None if deployment failed + """ # Skip cleanup when in dry-run mode dry_run = config.project.get_config("runtime.kserve.dry_run", False) @@ -570,19 +576,17 @@ def cleanup_test_resources() -> None: logger.info("Skipping cleanup_test_resources - dry-run mode enabled") return + if not llmisvc_name: + logger.warning("No LLMInferenceService name provided, cleanup may be incomplete") + return + namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] smoke = platform["smoke"] - # Build the actual inference service name (includes deployment profile) - base_name = inference_service["name"] - deployment_profile_name = runtime_config.get_deployment_profile_name() - actual_service_name = f"{base_name}-{deployment_profile_name}" - cleanup_test_resources_command.run( namespace=namespace, - inference_service_name=actual_service_name, + inference_service_name=llmisvc_name, smoke_pod_name=smoke["pod_name"], benchmark_job_name=runtime_config.get_benchmark_job_name(), ) From 03c63d6767d2e1203c623eb19287a8fb0ddc2ae5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 20:37:13 +0200 Subject: [PATCH 0313/1072] [llm_d] orchestration: test_phase: use the status_address_name only with a scheduler --- projects/llm_d/orchestration/test_phase.py | 98 +++++++++++++++++----- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c50d9958c..4876db1e7 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -339,29 +339,21 @@ def do_test() -> int: return 0 -def deploy_inference_service(llmisvc_name: str) -> str: - """Deploy LLMInferenceService and return endpoint URL and service name. +def _try_reuse_existing_service( + namespace: str, llmisvc_name: str, gateway: dict[str, str], reuse_existing: bool, dry_run: bool +) -> str | None: + """Try to reuse an existing LLMInferenceService if enabled and available. Args: - llmisvc_name: The name of the LLMInferenceService to deploy + namespace: Target namespace + llmisvc_name: Name of the LLMInferenceService + gateway: Gateway configuration + reuse_existing: Whether to attempt reuse + dry_run: Whether in dry-run mode Returns: - Gateway endpoint URL + Endpoint URL if reuse successful, None otherwise """ - logger.info("Starting LLMInferenceService deployment") - - # Load config where it's consumed - - namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - gateway = platform["gateway"] - - # llmisvc_name is now passed as a parameter - - dry_run = config.project.get_config("runtime.kserve.dry_run") - wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") - reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) - # Check if we should reuse existing LLMInferenceService if reuse_existing and not dry_run: logger.info(f"Checking if LLMInferenceService {llmisvc_name} already exists") @@ -383,12 +375,26 @@ def deploy_inference_service(llmisvc_name: str) -> str: ) # Import and use the toolbox function to extract URL + # Check if the existing service has a scheduler to determine gateway address + from projects.core.dsl.utils.k8s import oc_get_json from projects.kserve.toolbox.deploy_llmisvc import try_resolve_endpoint_url + existing_service = oc_get_json( + "llminferenceservice", name=llmisvc_name, namespace=namespace + ) + has_scheduler = ( + existing_service.get("spec", {}).get("router", {}).get("scheduler") is not None + ) + + # Use None for status_address_name when existing service has no scheduler + gateway_status_address_name = ( + gateway["status_address_name"] if has_scheduler else None + ) + endpoint_url = try_resolve_endpoint_url( namespace=namespace, inference_service_name=llmisvc_name, - gateway_status_address_name=gateway["status_address_name"], + gateway_status_address_name=gateway_status_address_name, ) if endpoint_url: @@ -408,6 +414,43 @@ def deploy_inference_service(llmisvc_name: str) -> str: f"Error checking for existing LLMInferenceService: {e}, proceeding with new deployment" ) + return None + + +def deploy_inference_service(llmisvc_name: str) -> str: + """Deploy LLMInferenceService and return endpoint URL and service name. + + Args: + llmisvc_name: The name of the LLMInferenceService to deploy + + Returns: + Gateway endpoint URL + """ + logger.info("Starting LLMInferenceService deployment") + + # Load config where it's consumed + + namespace = runtime_config.get_namespace() + platform = runtime_config.get_platform_config() + gateway = platform["gateway"] + + # llmisvc_name is now passed as a parameter + + dry_run = config.project.get_config("runtime.kserve.dry_run") + wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + # Try to reuse existing LLMInferenceService if enabled + endpoint_url = _try_reuse_existing_service( + namespace=namespace, + llmisvc_name=llmisvc_name, + gateway=gateway, + reuse_existing=reuse_existing, + dry_run=dry_run, + ) + if endpoint_url: + return endpoint_url + # Step 1: Ensure model cache is ready (skip in dry-run) if not dry_run: _prepare_model_cache() @@ -422,6 +465,21 @@ def deploy_inference_service(llmisvc_name: str) -> str: # Step 3: Build and write inference service manifest manifest_path = _build_inference_service_manifest() + # Check if the manifest has a scheduler to determine gateway status address name + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) + + # Use None for status_address_name when deploying without a scheduler + gateway_status_address_name = gateway["status_address_name"] if has_scheduler else None + # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) @@ -431,7 +489,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), - gateway_status_address_name=gateway["status_address_name"], + gateway_status_address_name=gateway_status_address_name, dry_run=dry_run, wait_pods_scheduled=wait_pods_scheduled, ) From a7e47b7005d9901f7bc8585e58f6fde05dc38b41 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 23 Jul 2026 22:38:45 +0200 Subject: [PATCH 0314/1072] [llm_d] orchestration: test_phase: correctly extract the llmisvc name --- projects/llm_d/orchestration/test_phase.py | 70 +++++++++++++++------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 4876db1e7..dbb0d1f6a 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -280,10 +280,10 @@ def do_test() -> int: cleanup_existing_resources(namespace) endpoint_url: str | None = None - llmisvc_name: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None + actual_llmisvc_name = "llmisvc-na-not-computed" try: # Create test labels with actual model and profile information create_test_labels() @@ -296,12 +296,16 @@ def do_test() -> int: inference_service = platform["inference_service"] base_name = inference_service["name"] deployment_profile_name = runtime_config.get_deployment_profile_name() - llmisvc_name = ( + # Step 1: Build manifest and get actual truncated name + initial_llmisvc_name = ( f"{base_name}-{deployment_profile_name}" if deployment_profile_name else base_name ) - llmisvc_name = slugify_identifier(llmisvc_name) + initial_llmisvc_name = slugify_identifier(initial_llmisvc_name) - endpoint_url = deploy_inference_service(llmisvc_name) + manifest_path, actual_llmisvc_name = build_inference_service_manifest(initial_llmisvc_name) + + # Step 2: Deploy using the actual name from the manifest + endpoint_url = deploy_inference_service_from_manifest(manifest_path, actual_llmisvc_name) if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") @@ -312,7 +316,7 @@ def do_test() -> int: run_smoke_request(endpoint_url=endpoint_url) run_guidellm_benchmark(endpoint_url=endpoint_url) - except Exception: + except Exception as e: primary_exc = sys.exc_info() except SignalInterrupt: primary_exc = sys.exc_info() @@ -327,7 +331,7 @@ def do_test() -> int: if do_finalizers: primary_exc, finalizer_exc = run_finalizers( - endpoint_url, llmisvc_name, primary_exc, finalizer_exc + endpoint_url, actual_llmisvc_name, primary_exc, finalizer_exc ) if primary_exc is not None: @@ -417,25 +421,52 @@ def _try_reuse_existing_service( return None -def deploy_inference_service(llmisvc_name: str) -> str: - """Deploy LLMInferenceService and return endpoint URL and service name. +def build_inference_service_manifest(llmisvc_name: str) -> tuple[Path, str]: + """Build inference service manifest and return path and actual service name. Args: - llmisvc_name: The name of the LLMInferenceService to deploy + llmisvc_name: The initial name for the LLMInferenceService + + Returns: + Tuple of (manifest_path, actual_llmisvc_name) where actual_llmisvc_name may be truncated + """ + # Step 1: Build and write inference service manifest + manifest_path = _build_inference_service_manifest() + + # Step 2: Extract the actual deployed name from the manifest (may be truncated) + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + # Get the actual deployed name from the manifest (may be truncated) + actual_llmisvc_name = manifest["metadata"]["name"] + + logger.info( + "Built LLMInferenceService manifest: %s (actual name: %s)", + manifest_path, + actual_llmisvc_name, + ) + return manifest_path, actual_llmisvc_name + + +def deploy_inference_service_from_manifest(manifest_path: Path, actual_llmisvc_name: str) -> str: + """Deploy LLMInferenceService from pre-built manifest and return endpoint URL. + + Args: + manifest_path: Path to the pre-built manifest + actual_llmisvc_name: The actual name of the LLMInferenceService from the manifest Returns: Gateway endpoint URL """ - logger.info("Starting LLMInferenceService deployment") + logger.info("Starting LLMInferenceService deployment from manifest") # Load config where it's consumed - namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # llmisvc_name is now passed as a parameter - dry_run = config.project.get_config("runtime.kserve.dry_run") wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) @@ -443,7 +474,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: # Try to reuse existing LLMInferenceService if enabled endpoint_url = _try_reuse_existing_service( namespace=namespace, - llmisvc_name=llmisvc_name, + llmisvc_name=actual_llmisvc_name, gateway=gateway, reuse_existing=reuse_existing, dry_run=dry_run, @@ -462,10 +493,7 @@ def deploy_inference_service(llmisvc_name: str) -> str: rhoai_namespace = platform["rhoai"]["namespace"] wait_kserve_ready.run(namespace=rhoai_namespace) - # Step 3: Build and write inference service manifest - manifest_path = _build_inference_service_manifest() - - # Check if the manifest has a scheduler to determine gateway status address name + # Step 3: Check manifest for scheduler to determine gateway status address name import yaml with manifest_path.open(encoding="utf-8") as f: @@ -495,10 +523,8 @@ def deploy_inference_service(llmisvc_name: str) -> str: ) if dry_run: - llmisvc_manifest_path = endpoint_url - logger.info("Dry-run completed: LLMInferenceService manifest prepared:") - logger.info(llmisvc_manifest_path) - return llmisvc_manifest_path, llmisvc_name + logger.info("Dry-run completed: LLMInferenceService manifest prepared: %s", manifest_path) + return str(manifest_path) logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url From 23ec952016c116d7a4a305a80ea40d5469da0c6f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:55:29 +0200 Subject: [PATCH 0315/1072] [llm_d] orchestration: config: enable the KPI export --- projects/llm_d/orchestration/config.yaml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 76bc188dc..3ced4607f 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -59,18 +59,13 @@ caliper: enabled: true output: kpis/kpis.json kpis_to_csv: - enabled: false + enabled: true output: kpis/kpis.csv include_header_comments: true artifacts_to_ai_data: enabled: true output_dir: ai_data - analyze: - enabled: false - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke @@ -93,9 +88,11 @@ caliper: include_kpis_json: true include_ai_data: true - analyse_kpis: - enabled: false - output: kpi_analysis.json + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json agentic: enabled: true From 66dbd08c6192bf7579a37e0f847943f89ac19d6f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 11:58:52 +0200 Subject: [PATCH 0316/1072] [llm_d] orchestration: test_phase: expose the test completion status in the __test_labels__.yaml --- projects/llm_d/orchestration/test_phase.py | 41 +++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index dbb0d1f6a..b8c25fb67 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -157,6 +157,38 @@ def create_test_labels() -> None: logger.info("Created test labels: %s", labels) +def update_test_labels_with_status(success: bool, message: str) -> None: + """Update __test_labels__.yaml with test execution status. + + Args: + success: True if test succeeded, False if failed + message: Status message describing the result + """ + test_labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + + # Read existing labels + if test_labels_path.exists(): + with test_labels_path.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) + else: + # Fallback if labels file doesn't exist + data = {"version": "1", "labels": {}} + + # Add completion information as separate top-level field + data["completion"] = { + "success": success, + "message": message, + } + + # Write updated labels + with test_labels_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(data, f, sort_keys=False) + + logger.info( + "Updated test labels with completion status: success=%s, message=%s", success, message + ) + + def run_all_tests(stop_on_error: bool = False) -> int: """Run tests for all run specifications without post-processing. @@ -183,6 +215,7 @@ def run_all_tests(stop_on_error: bool = False) -> int: return exit_code except Exception as e: logger.exception(f"Test failed with exception: {e}") + # Note: Status update already handled by do_test() exception handler max_exit_code = 1 if stop_on_error: logger.error("Stopping due to stop_on_error") @@ -309,6 +342,7 @@ def do_test() -> int: if dry_run: logging.warning("Running in dry-run mode, skipping the rest of the test steps") + update_test_labels_with_status(True, "Dry-run completed successfully") return 0 if not endpoint_url: @@ -318,8 +352,10 @@ def do_test() -> int: run_guidellm_benchmark(endpoint_url=endpoint_url) except Exception as e: primary_exc = sys.exc_info() - except SignalInterrupt: + update_test_labels_with_status(False, f"Test failed with exception: {str(e)}") + except SignalInterrupt as e: primary_exc = sys.exc_info() + update_test_labels_with_status(False, f"Test interrupted: {str(e)}") finally: do_finalizers = config.project.get_config("runtime.run_test_finalizers") if primary_exc and isinstance(primary_exc[1], SignalInterrupt): @@ -340,6 +376,9 @@ def do_test() -> int: if finalizer_exc is not None: raise finalizer_exc[1].with_traceback(finalizer_exc[2]) + # Update test labels with success status + update_test_labels_with_status(True, "Test completed successfully") + return 0 From e973bdbc42807e5b6ac5c13f0605d8328611cc3a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:07:25 +0200 Subject: [PATCH 0317/1072] [llm_d] orchestration: disable Kueue when necessary --- .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../llm_d/orchestration/presets.d/cks.yaml | 1 + .../orchestration/render_inference_service.py | 50 ++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 996064747..bea48e83a 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -15,6 +15,7 @@ kueue: enabled: false queue_name: default-queue local_queue_manifest: manifests/kueue-localqueue.yaml + disable_above_n_gpus: null router: template: file: manifests/scheduler_config/__router_template__.yaml diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index e96a453c3..78cc89024 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -7,6 +7,7 @@ cks: runtime.kserve.wait_readiness: false runtime.kueue.enabled: true runtime.kueue.queue_name: llm-d-cluster-queue + runtime.kueue.disable_above_n_gpus: 8 # Use ReadWriteMany for model cache PVC in CKS environment model_cache.pvc.access_mode: ReadWriteMany diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 2daee9e10..d71e0b22c 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -2,6 +2,7 @@ import copy import hashlib +import logging from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.library import config +logger = logging.getLogger(__name__) + def _load_yaml(path: Path) -> Any: with path.open(encoding="utf-8") as handle: @@ -124,7 +127,7 @@ def render_inference_service_from_parts( ) # Apply Kueue configuration if enabled - _apply_kueue_configuration(rendered_manifest) + _apply_kueue_configuration(rendered_manifest, deployment_profile) return rendered_manifest @@ -405,11 +408,45 @@ def _build_pd_pod_template( return {"containers": [container]} -def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: +def _calculate_total_gpu_usage(deployment_profile: dict[str, Any]) -> int: + """Calculate the total GPU usage for a deployment profile. + + Args: + deployment_profile: The deployment profile configuration + + Returns: + Total number of GPUs required for this deployment + """ + # Check if this is a P/D deployment (has prefill/decode sections) + if "prefill" in deployment_profile and "decode" in deployment_profile: + # P/D deployment: sum up prefill and decode GPU usage + prefill_gpus = ( + deployment_profile["prefill"]["tensor_parallelism"] + * deployment_profile["prefill"]["replicas"] + ) + decode_gpus = ( + deployment_profile["decode"]["tensor_parallelism"] + * deployment_profile["decode"]["replicas"] + ) + return prefill_gpus + decode_gpus + else: + # Standard deployment: tensor_parallelism * replicas + tensor_parallelism = deployment_profile.get("tensor_parallelism", 1) + replicas = deployment_profile.get("replicas", 1) + return tensor_parallelism * replicas + + +def _apply_kueue_configuration( + manifest: dict[str, Any], deployment_profile: dict[str, Any] +) -> None: """Apply Kueue annotations and labels to the ISVC manifest. Based on the implementation from topsail's test_llmd.py. Can be enabled by setting runtime.kserve_use_kueue config. + + Args: + manifest: The Kubernetes manifest to modify + deployment_profile: The deployment profile configuration used to calculate GPU usage """ # Check if kueue annotations should be enabled enable_kueue = config.project.get_config("runtime.kueue.enabled") @@ -417,6 +454,15 @@ def _apply_kueue_configuration(manifest: dict[str, Any]) -> None: if not enable_kueue: return + # Calculate total GPU usage from deployment profile + total_gpus = _calculate_total_gpu_usage(deployment_profile) + + # Check if we should skip Kueue due to high GPU usage + disable_above_n_gpus = config.project.get_config("runtime.kueue.disable_above_n_gpus") + if disable_above_n_gpus is not None and total_gpus > disable_above_n_gpus: + logger.info(f"Skipping Kueue labels: {total_gpus} GPUs > {disable_above_n_gpus} threshold") + return + # Configure kueue settings queue_name = config.project.get_config("runtime.kueue.queue_name") kueue_config = { From c25eed5a22316848bd024f09ceaa6e173719e47e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 12:12:47 +0200 Subject: [PATCH 0318/1072] [llm_d] orchestration: save the config and fjob --- projects/llm_d/orchestration/test_phase.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index b8c25fb67..7ba815fbc 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -1,12 +1,14 @@ from __future__ import annotations import logging +import shutil import sys from pathlib import Path from typing import Any import yaml +from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier from projects.core.dsl.utils.k8s import oc @@ -156,6 +158,27 @@ def create_test_labels() -> None: write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) + # Dump config.project to config.yaml + config_path = env.ARTIFACT_DIR / "config.yaml" + try: + with config_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(config.project.config, f, sort_keys=False) + logger.info("Saved project configuration to: %s", config_path) + except Exception as e: + logger.warning("Failed to save project configuration: %s", e) + + # Copy fournos_fjob.yaml if available + fournos_source = env.ARTIFACT_DIR / CI_METADATA_DIRNAME / "fournos_fjob.yaml" + if fournos_source.exists(): + fournos_dest = env.ARTIFACT_DIR / "fournos_fjob.yaml" + try: + shutil.copy2(fournos_source, fournos_dest) + logger.info("Copied fournos job file: %s -> %s", fournos_source, fournos_dest) + except Exception as e: + logger.warning("Failed to copy fournos job file: %s", e) + else: + logger.debug("No fournos job file found at: %s", fournos_source) + def update_test_labels_with_status(success: bool, message: str) -> None: """Update __test_labels__.yaml with test execution status. From 20c57bbcf5b92036287e5d4a2b578f0ccb9148ea Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 16:11:05 +0200 Subject: [PATCH 0319/1072] [llm_d] orchestration: render_inference_service: correctly set the TP level --- projects/llm_d/orchestration/render_inference_service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index d71e0b22c..3e869d5ef 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -255,6 +255,7 @@ def _render_standard_deployment( name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name manifest["spec"]["replicas"] = deployment_profile["replicas"] + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["tensor_parallelism"]} serving_container = manifest["spec"]["template"]["containers"][0] serving_container["resources"] = _build_serving_resources(deployment_profile) @@ -308,6 +309,7 @@ def _render_pd_deployment( # Configure prefill section manifest["spec"]["prefill"] = { "replicas": get_prefill_pod_count(), + "parallelism": {"tensor": deployment_profile["prefill"]["tensor_parallelism"]}, "template": _build_pd_pod_template( deployment_profile, deployment_profile_name, is_prefill=True, workload=workload ), @@ -315,6 +317,7 @@ def _render_pd_deployment( # Configure main template (decode) manifest["spec"]["replicas"] = get_decode_pod_count() + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["decode"]["tensor_parallelism"]} manifest["spec"]["template"] = _build_pd_pod_template( deployment_profile, deployment_profile_name, is_prefill=False, workload=workload ) From 1d0b97db44635198730320bb93692f1276004cb0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:09:40 +0200 Subject: [PATCH 0320/1072] [llm_d] orchestration: ci: make the RHOAI vaults mandatory if the custom catalog is enabled --- projects/llm_d/orchestration/ci.py | 2 +- projects/llm_d/tests/test_profiles.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index de85604bc..cb0adc76d 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -69,7 +69,7 @@ def init_vaults_for_phase(phase: str) -> None: if phase == "prepare" and config.project.get_config( "platform.rhoai.custom_catalog.enabled", False ): - optional_vaults = [*optional_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] + mandatory_vaults = [*mandatory_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 9409efce6..1efce2f07 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -233,10 +233,10 @@ def _fake_init(**kwargs): core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) llmd_ci.init_vaults_for_phase("prepare") - assert "psap-rhoai-rc" not in calls[0]["optional_vaults"] - assert "psap-forge-staging-image-pull" not in calls[0]["optional_vaults"] - assert "psap-rhoai-rc" in calls[1]["optional_vaults"] - assert "psap-forge-staging-image-pull" in calls[1]["optional_vaults"] + assert "psap-rhoai-rc" not in calls[0]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" not in calls[0]["mandatory_vaults"] + assert "psap-rhoai-rc" in calls[1]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" in calls[1]["mandatory_vaults"] def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( From 66eb60def1ade714e415a2a3b0d66b1d18af7b58 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 18:25:14 +0200 Subject: [PATCH 0321/1072] [llm_d] orchestration: presets: rhoai-rc: new preset for RHOAI RC --- projects/llm_d/orchestration/presets.d/rhoai-rc.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 projects/llm_d/orchestration/presets.d/rhoai-rc.yaml diff --git a/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml new file mode 100644 index 000000000..d1943a32c --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml @@ -0,0 +1,7 @@ +rhoai-rc: + platform.rhoai.custom_catalog.enabled: true + platform.operators.rhods-operator.channel: beta + +rhoai-3.5-ea.2: + extends: [rhoai-rc] + platform.rhoai.custom_catalog.image: quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310 From e935a3ba07706c646e5fc01c22a29b709ef8e531 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 21:54:11 +0200 Subject: [PATCH 0322/1072] [llm_d] tests: reference_deployments: correctly set the TP --- .../deployment-intelligentrouting-tp4-x4/llmisvc.yaml | 2 ++ .../deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml | 4 ++++ .../deployment-simple-tp4-x4/llmisvc.yaml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml index c4a16a88b..42aa4b492 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -60,3 +60,5 @@ spec: - name: VLLM_ADDITIONAL_ARGS value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index 8dbe6a352..b9cc3dee3 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -45,6 +45,8 @@ spec: fieldPath: status.podIP prefill: replicas: 8 + parallelism: + tensor: 1 template: containers: - name: main @@ -65,3 +67,5 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml index f17b89144..a9e9aeb1b 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -51,3 +51,5 @@ spec: - name: VLLM_ADDITIONAL_ARGS value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 From 865e50fdfdf6c6e93ecfd1ce8ddd00f181c37791 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 22:08:28 +0200 Subject: [PATCH 0323/1072] [llm_d] orchestration: presets: cpt: more presets --- projects/llm_d/orchestration/presets.d/cks.yaml | 2 ++ projects/llm_d/orchestration/presets.d/cpt.yaml | 3 +++ projects/llm_d/orchestration/presets.d/janus.yaml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index 78cc89024..1b59f614b 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -20,6 +20,7 @@ cks: cks-intelligent: extends: - cks + - cpt - llama3-70b runtime.deployment_profile: @@ -33,6 +34,7 @@ cks-intelligent: cks-pd: extends: - cks + - cpt - gpt-oss runtime.deployment_profile: diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index dfb731711..ce7ce6859 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -8,3 +8,6 @@ cpt-reference-flavors: - pd-d.x2-p.tp1-d.tp4-p.x8 - intelligentrouting-tp4-x4 - simple-tp4-x4 + +cpt: + caliper.export.backend.mlflow.config.experiment: cpt-llm-d diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml index 4026bc346..9612228ce 100644 --- a/projects/llm_d/orchestration/presets.d/janus.yaml +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -13,6 +13,7 @@ janus: janus-intelligent: extends: - janus + - cpt - llama3-70b runtime.deployment_profile: @@ -26,6 +27,7 @@ janus-intelligent: janus-pd: extends: - janus + - cpt - gpt-oss runtime.deployment_profile: From b35cc93342f51478bc834ced34889b20873ed5b6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 07:40:51 +0200 Subject: [PATCH 0324/1072] [llm_d] orchestration: render_inference_service: add anti-affinity for prefill/decode pods --- .../orchestration/render_inference_service.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 3e869d5ef..b27456e9c 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -408,7 +408,38 @@ def _build_pd_pod_template( if deployment_profile.get("serving_image"): container["image"] = deployment_profile["serving_image"] - return {"containers": [container]} + # Build pod template with anti-affinity for P/D deployments + component_type = "prefill" if is_prefill else "decode" + opposite_component = "decode" if is_prefill else "prefill" + + pod_template = { + "containers": [container], + "metadata": {"labels": {"app.kubernetes.io/component": component_type}}, + } + + # Add anti-affinity to prevent prefill and decode pods from landing on the same node + affinity = { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 100, + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + # Anti-affinity between prefill and decode pods + "app.kubernetes.io/component": opposite_component + } + }, + "topologyKey": "kubernetes.io/hostname", + }, + } + ] + } + } + + pod_template["affinity"] = affinity + + return pod_template def _calculate_total_gpu_usage(deployment_profile: dict[str, Any]) -> int: From 25c7083ae3871f2ad2f86e5db6e75dd922c69678 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 09:42:01 +0200 Subject: [PATCH 0325/1072] [llm_d] tests: reference_deployments: add the anti-affinity fields --- .../llmisvc.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml index b9cc3dee3..a12df366c 100644 --- a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -43,6 +43,18 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: decode + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: prefill + topologyKey: kubernetes.io/hostname prefill: replicas: 8 parallelism: @@ -67,5 +79,17 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: prefill + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: decode + topologyKey: kubernetes.io/hostname parallelism: tensor: 4 From d70c86df2e2147c19a5672e2ce90ee901f504516 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 10:37:28 +0200 Subject: [PATCH 0326/1072] Revert Revert '[caliper] remove unnecessary reference to the hierachical format' --- projects/caliper/engine/kpi/analyze.py | 349 +++++++++++++++++++++++-- 1 file changed, 330 insertions(+), 19 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 246c317e5..47ae439ff 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -2,32 +2,343 @@ from __future__ import annotations +import json +import logging +import time +from pathlib import Path from typing import Any -from projects.caliper.engine.model import RegressionFinding +logger = logging.getLogger(__name__) + + +def run_kpi_analysis( + current_kpi_file: Path, + historical_data_dir: Path, + output_file: Path, + plugin_module: str, +) -> int: + """Run KPI analysis and generate output file. + + Args: + current_kpi_file: Path to current KPI JSON file + historical_data_dir: Directory containing historical KPI files + output_file: Path where analysis results will be written + plugin_module: Plugin module name for analysis + + Returns: + Exit code (0 for success, non-zero for failure) + """ + try: + logger.info(f"Running KPI analysis for: {current_kpi_file}") + logger.info(f"Historical data directory: {historical_data_dir}") + logger.info(f"Output file: {output_file}") + logger.info(f"Plugin module: {plugin_module}") + + # Check if current KPI file exists + if not current_kpi_file.exists(): + logger.error(f"Current KPI file not found: {current_kpi_file}") + return 1 + + # Check if historical data directory exists + if not historical_data_dir.exists(): + logger.error(f"Historical data directory not found: {historical_data_dir}") + return 1 + + # Find all historical KPI files + historical_kpi_files = [] + for kpi_file in historical_data_dir.rglob("kpis.json"): + historical_kpi_files.append(str(kpi_file)) + logger.info(f"Found historical KPI file: {kpi_file}") + + if not historical_kpi_files: + logger.warning("No historical KPI files found for analysis") + # Return special exit code 2 to indicate warning (no historical data) + analysis_result = { + "status": "warning", + "message": "no historical KPI found for regression testing", + "current_kpi_file": str(current_kpi_file), + "historical_kpi_files": historical_kpi_files, + "baseline_files_count": 0, + "plugin_module": plugin_module, + "completed_at": time.time(), + } + # Write warning result to output file + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis_result, f, indent=2) + return 2 # Special exit code for warning + + logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") + + # Create analysis result + analysis_result = { + "status": "success (stub)", + "current_kpi_file": str(current_kpi_file), + "historical_kpi_files": historical_kpi_files, + "baseline_files_count": len(historical_kpi_files), + "plugin_module": plugin_module, + "completed_at": time.time(), + } + + # Create output directory if needed + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Write analysis results to output file + with open(output_file, "w") as f: + json.dump(analysis_result, f, indent=2) + f.write("\n") # Add EOL at EOF + + logger.info(f"Analysis completed successfully, results written to: {output_file}") + return 0 + + except Exception as e: + logger.exception("KPI analysis failed") + + # Create failure result + failure_result = { + "status": "failed", + "error": str(e), + "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, + "completed_at": time.time(), + } + + try: + # Attempt to write failure result + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(failure_result, f, indent=2) + f.write("\n") # Add EOL at EOF + except Exception: + logger.exception("Failed to write error result to output file") + + return 1 + + +def analyze_kpis( + postprocess_config, # CaliperOrchestrationPostprocessConfig + plugin_module: str, + base_dir: Path, + output_dir: Path, + current_kpis_file: Path, +) -> dict[str, Any]: + """Run KPI analysis step and return result status. + + This is the orchestration interface for KPI analysis. + """ + if not postprocess_config.analyze.enabled: + return {"status": "disabled", "reason": "analyze disabled"} + + # Determine paths for analysis + analyze_config = postprocess_config.analyze + + # Historical KPIs directory path + historical_kpis_dir = Path(analyze_config.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Output path for analysis results + output_path = output_dir / analyze_config.output + + try: + # Check if required files exist + if not current_kpis_file.exists(): + return { + "status": "failed", + "error": f"Current KPI file not found: {current_kpis_file}", + "completed_at": time.time(), + } + + if not historical_kpis_dir.exists(): + return { + "status": "failed", + "error": f"Historical KPIs directory not found: {historical_kpis_dir}", + "completed_at": time.time(), + } + + # Call the core analysis function + exit_code = run_kpi_analysis( + current_kpi_file=current_kpis_file, + historical_data_dir=historical_kpis_dir, + output_file=output_path, + plugin_module=plugin_module, + ) + + if exit_code == 0: + # Read the results from the output file + if output_path.exists(): + with open(output_path) as f: + result_data = json.load(f) + + # Return success with output file path for notifications + return { + "status": "success", + "output_file": str(output_path), + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": "Analysis completed but no output file was generated", + "completed_at": time.time(), + } + elif exit_code == 2: + # Warning: no historical data found + if output_path.exists(): + with open(output_path) as f: + result_data = json.load(f) + + # Return warning status with message + return { + "status": "warning", + "message": result_data.get( + "message", "no historical KPI found for regression testing" + ), + "output_file": str(output_path), + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": time.time(), + } + else: + return { + "status": "warning", + "message": "no historical KPI found for regression testing", + "baseline_files_count": 0, + "completed_at": time.time(), + } + else: + # Read error from output file if it exists + error_msg = f"Analysis failed with exit code {exit_code}" + if output_path.exists(): + try: + with open(output_path) as f: + result_data = json.load(f) + error_msg = result_data.get("error", error_msg) + except Exception: + pass + + return { + "status": "failed", + "error": error_msg, + "completed_at": time.time(), + } + + except Exception as e: + error_msg = f"{type(e).__name__}: {str(e)}" + logger.exception(f"Analysis step failed with {error_msg}") + + # Don't suppress the exception - provide detailed error info and re-raise + import traceback + + full_traceback = traceback.format_exc() + logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") + + # Re-raise the original exception so the caller can see the full details + raise RuntimeError(f"KPI analysis failed: {error_msg}") from e + + +def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: + """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. + + Args: + historical_dir: Directory to search for historical kpis.json files + + Returns: + Dictionary mapping file paths to loaded KPI JSON objects + """ + baseline_kpis = {} + kpi_files = list(historical_dir.rglob("kpis.json")) + + if not kpi_files: + logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") + return baseline_kpis + + logger.info(f"Found {len(kpi_files)} historical KPI files to load") + + for kpi_file in kpi_files: + try: + with open(kpi_file) as f: + kpi_data = json.load(f) + + # Validate that it's a hierarchical format (schema_version 2) + schema_version = kpi_data.get("schema_version", "unknown") + if schema_version != "2": + logger.warning( + f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" + ) + continue + + baseline_kpis[kpi_file] = kpi_data + logger.debug(f"Loaded KPI file: {kpi_file}") + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON in {kpi_file}: {e}") + continue + except FileNotFoundError as e: + logger.error(f"KPI file not found: {kpi_file}: {e}") + continue + except Exception as e: + logger.error(f"Failed to load KPI file {kpi_file}: {e}") + continue + + logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") + return baseline_kpis def run_analyze( *, current_path: Any, - baseline_path: Any, + baseline_kpis: dict[Path, dict[str, Any]], output_path: Any, plugin: Any = None, -) -> list[RegressionFinding]: - """Run KPI analysis - hierarchical format only.""" - from pathlib import Path - - from .analyze_hierarchical import analyze_hierarchical_kpis - - result = analyze_hierarchical_kpis( - current_kpis_path=Path(current_path), - baseline_kpis_path=Path(baseline_path), - output_path=Path(output_path), - plugin=plugin, - ) - - # Convert to expected format for CLI - if result["status"] == "success": - return [] # Return empty list for now, actual findings are in the output file +) -> dict[str, Any]: + """Run KPI analysis against ALL baseline files (CLI interface).""" + # For CLI use, we need to extract the baseline directory from the dict + # This is a bit of a hack, but the CLI interface expects a directory + if baseline_kpis: + # Get the parent directory of the first baseline file + first_baseline_path = next(iter(baseline_kpis.keys())) + historical_dir = first_baseline_path.parent + + # Extract plugin module name if available + plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" + + # Call the core analysis function + exit_code = run_kpi_analysis( + current_kpi_file=Path(current_path), + historical_data_dir=historical_dir, + output_file=Path(output_path), + plugin_module=plugin_module, + ) + + if exit_code == 0: + # Read results and convert to expected format for CLI + try: + with open(Path(output_path)) as f: + result_data = json.load(f) + return { + "status": "success", + "baseline_files_count": result_data.get("baseline_files_count", 0), + "completed_at": result_data.get("completed_at", time.time()), + } + except Exception as e: + return { + "status": "failed", + "error": f"Failed to read analysis results: {e}", + "completed_at": time.time(), + } + else: + return { + "status": "failed", + "error": f"Analysis failed with exit code {exit_code}", + "completed_at": time.time(), + } else: - raise RuntimeError(f"Analysis failed: {result.get('error', 'Unknown error')}") + return { + "status": "failed", + "error": "No baseline KPI files provided", + "completed_at": time.time(), + } + + +# EOF From bee915b053340b338ff4c4a7d849ec4858480a36 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 01:06:52 -0400 Subject: [PATCH 0327/1072] feat(caliper): implement KPI regression analysis POC Replace the stub run_kpi_analysis with a working regression engine that: - Matches current KPIs to historical baselines via configurable comparison_keys/ignored_keys on record labels - Runs per-KPI regression tests (relative threshold, higher/lower-is-better) - Produces a structured YAML report with processed/tested/results/overall - Supports plugin-provided AnalysisConfig for per-project tuning Exit codes: 0=pass, 1=error, 2=no-baseline, 3=regression-detected. Includes end-to-end test suite (15 tests). Co-authored-by: Cursor --- projects/caliper/engine/kpi/analyze.py | 616 ++++++++++++++------- projects/caliper/tests/test_kpi_analyze.py | 442 +++++++++++++++ 2 files changed, 855 insertions(+), 203 deletions(-) create mode 100644 projects/caliper/tests/test_kpi_analyze.py diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 47ae439ff..593f8acdb 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -1,120 +1,406 @@ -"""Generic KPI analysis against historical baselines.""" +"""Generic KPI regression analysis against historical baselines.""" from __future__ import annotations import json import logging import time +from dataclasses import dataclass, field from pathlib import Path from typing import Any +import yaml + logger = logging.getLogger(__name__) +@dataclass +class AnalysisConfig: + """Configuration for KPI regression analysis. + + comparison_keys: Label keys that define what we compare against. + Records must differ on at least one comparison key to be distinct baselines. + E.g. ["version"] means we test the current version against other versions. + ignored_keys: Label keys excluded when matching current to baseline records. + E.g. ["os"] means we match across operating systems. + sorting_keys: Label keys used to order entries in the output report. + max_relative_regression: Fraction threshold for flagging regression (0.1 = 10%). + min_baseline_points: Minimum number of baseline data points required to run a test. + """ + + comparison_keys: list[str] = field(default_factory=list) + ignored_keys: list[str] = field(default_factory=list) + sorting_keys: list[str] = field(default_factory=list) + max_relative_regression: float = 0.1 + min_baseline_points: int = 1 + + +@dataclass +class KpiTestResult: + """Result of a single KPI regression test.""" + + kpi_id: str + labels: dict[str, Any] + current_value: float + baseline_mean: float + relative_change: float + higher_is_better: bool + regression: bool + baseline_count: int + + +@dataclass +class AnalysisReport: + """Full analysis report.""" + + status: str # "success", "no_regression", "regression_detected", "error" + processed: dict[str, Any] + tested: list[dict[str, Any]] + results: list[dict[str, Any]] + overall: dict[str, Any] + + +def _load_analysis_config(plugin_module: str) -> AnalysisConfig: + """Load analysis config from plugin module if available, else use defaults. + + Plugins can expose an `analysis_config` dict or `get_analysis_config()` callable. + """ + try: + mod = __import__(plugin_module, fromlist=[""]) + if hasattr(mod, "get_analysis_config"): + raw = mod.get_analysis_config() + elif hasattr(mod, "analysis_config"): + raw = mod.analysis_config + else: + return AnalysisConfig() + + if isinstance(raw, AnalysisConfig): + return raw + if isinstance(raw, dict): + return AnalysisConfig( + **{k: v for k, v in raw.items() if k in AnalysisConfig.__dataclass_fields__} + ) + except (ImportError, Exception) as exc: + logger.debug("Could not load analysis config from plugin %s: %s", plugin_module, exc) + + return AnalysisConfig() + + +def _match_key( + labels: dict[str, Any], ignored_keys: list[str], comparison_keys: list[str] +) -> tuple: + """Build a hashable match key from labels, excluding ignored and comparison keys.""" + excluded = set(ignored_keys) | set(comparison_keys) + return tuple(sorted((k, str(v)) for k, v in labels.items() if k not in excluded)) + + +def _extract_kpi_records_from_hierarchical(data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract flat KPI records from a schema_version=2 hierarchical document.""" + records = [] + for test in data.get("tests", []): + test_labels = test.get("labels", {}) + for kpi in test.get("kpis", []): + records.append( + { + "kpi_id": kpi.get("id"), + "value": kpi.get("value"), + "unit": kpi.get("unit"), + "higher_is_better": kpi.get("higher_is_better", True), + "labels": test_labels, + "run_id": test.get("run_id"), + "metadata": test.get("metadata", {}), + } + ) + return records + + +def _build_baseline_index( + baseline_kpi_data: dict[Path, dict[str, Any]], + config: AnalysisConfig, +) -> dict[tuple, list[dict[str, Any]]]: + """Index baseline records by (kpi_id, match_key) for fast lookup. + + Returns mapping from (kpi_id, match_key) -> list of baseline records. + """ + index: dict[tuple, list[dict[str, Any]]] = {} + for _path, data in baseline_kpi_data.items(): + records = _extract_kpi_records_from_hierarchical(data) + for rec in records: + kpi_id = rec.get("kpi_id") + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + index.setdefault(key, []).append(rec) + return index + + +def _run_regression_test( + current: dict[str, Any], + baselines: list[dict[str, Any]], + config: AnalysisConfig, +) -> KpiTestResult: + """Run a regression test for a single KPI record against its baselines.""" + current_value = float(current["value"]) + higher_is_better = current.get("higher_is_better", True) + baseline_values = [float(b["value"]) for b in baselines if b.get("value") is not None] + baseline_mean = sum(baseline_values) / len(baseline_values) + + if baseline_mean == 0: + relative_change = 0.0 + else: + relative_change = (current_value - baseline_mean) / abs(baseline_mean) + + # Determine regression: if higher is better, a negative change is regression. + # If lower is better, a positive change is regression. + if higher_is_better: + regression = relative_change < -config.max_relative_regression + else: + regression = relative_change > config.max_relative_regression + + return KpiTestResult( + kpi_id=current["kpi_id"], + labels=current.get("labels", {}), + current_value=current_value, + baseline_mean=round(baseline_mean, 6), + relative_change=round(relative_change, 6), + higher_is_better=higher_is_better, + regression=regression, + baseline_count=len(baseline_values), + ) + + +def _sort_results(results: list[KpiTestResult], sorting_keys: list[str]) -> list[KpiTestResult]: + """Sort results by sorting keys extracted from labels, then by kpi_id.""" + + def sort_key(r: KpiTestResult): + label_key = tuple(str(r.labels.get(k, "")) for k in sorting_keys) + return (*label_key, r.kpi_id) + + return sorted(results, key=sort_key) + + +def _build_yaml_report( + results: list[KpiTestResult], + config: AnalysisConfig, + current_source: str, + baseline_sources: list[str], + skipped: list[dict[str, Any]], +) -> dict[str, Any]: + """Build the final YAML-serializable report structure.""" + regressions = [r for r in results if r.regression] + improvements = [r for r in results if not r.regression and r.relative_change != 0] + + if regressions: + overall_status = "REGRESSION_DETECTED" + else: + overall_status = "PASS" + + report = { + "analysis": { + "status": overall_status, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "config": { + "comparison_keys": config.comparison_keys, + "ignored_keys": config.ignored_keys, + "sorting_keys": config.sorting_keys, + "max_relative_regression": config.max_relative_regression, + "min_baseline_points": config.min_baseline_points, + }, + }, + "processed": { + "current_source": current_source, + "baseline_sources": baseline_sources, + "baseline_source_count": len(baseline_sources), + }, + "tested": { + "total_kpis": len(results), + "regressions": len(regressions), + "passes": len(results) - len(regressions), + "skipped": len(skipped), + }, + "results": [ + { + "kpi_id": r.kpi_id, + "labels": r.labels, + "current_value": r.current_value, + "baseline_mean": r.baseline_mean, + "relative_change_pct": round(r.relative_change * 100, 2), + "higher_is_better": r.higher_is_better, + "verdict": "REGRESSION" if r.regression else "PASS", + "baseline_count": r.baseline_count, + } + for r in results + ], + "overall": { + "verdict": overall_status, + "regression_count": len(regressions), + "total_tested": len(results), + "total_skipped": len(skipped), + }, + } + + if skipped: + report["skipped"] = skipped + + return report + + def run_kpi_analysis( current_kpi_file: Path, historical_data_dir: Path, output_file: Path, plugin_module: str, ) -> int: - """Run KPI analysis and generate output file. + """Run KPI regression analysis and generate a YAML report. Args: - current_kpi_file: Path to current KPI JSON file - historical_data_dir: Directory containing historical KPI files - output_file: Path where analysis results will be written - plugin_module: Plugin module name for analysis + current_kpi_file: Path to current KPI JSON file (hierarchical schema v2) + historical_data_dir: Directory containing historical KPI files (kpis.json) + output_file: Path where YAML analysis report will be written + plugin_module: Plugin module name (for loading analysis config) Returns: - Exit code (0 for success, non-zero for failure) + Exit code: 0=pass, 1=error, 2=warning (no baseline), 3=regression detected """ try: - logger.info(f"Running KPI analysis for: {current_kpi_file}") - logger.info(f"Historical data directory: {historical_data_dir}") - logger.info(f"Output file: {output_file}") - logger.info(f"Plugin module: {plugin_module}") + logger.info("Running KPI regression analysis") + logger.info(" current_kpi_file: %s", current_kpi_file) + logger.info(" historical_data_dir: %s", historical_data_dir) + logger.info(" output_file: %s", output_file) + logger.info(" plugin_module: %s", plugin_module) - # Check if current KPI file exists if not current_kpi_file.exists(): - logger.error(f"Current KPI file not found: {current_kpi_file}") + logger.error("Current KPI file not found: %s", current_kpi_file) return 1 - # Check if historical data directory exists if not historical_data_dir.exists(): - logger.error(f"Historical data directory not found: {historical_data_dir}") + logger.error("Historical data directory not found: %s", historical_data_dir) return 1 - # Find all historical KPI files - historical_kpi_files = [] - for kpi_file in historical_data_dir.rglob("kpis.json"): - historical_kpi_files.append(str(kpi_file)) - logger.info(f"Found historical KPI file: {kpi_file}") + config = _load_analysis_config(plugin_module) + logger.info( + " config: comparison_keys=%s, ignored_keys=%s", + config.comparison_keys, + config.ignored_keys, + ) - if not historical_kpi_files: - logger.warning("No historical KPI files found for analysis") - # Return special exit code 2 to indicate warning (no historical data) - analysis_result = { - "status": "warning", - "message": "no historical KPI found for regression testing", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": 0, - "plugin_module": plugin_module, - "completed_at": time.time(), - } - # Write warning result to output file - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - return 2 # Special exit code for warning - - logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") - - # Create analysis result - analysis_result = { - "status": "success (stub)", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": len(historical_kpi_files), - "plugin_module": plugin_module, - "completed_at": time.time(), - } + # Load current KPIs + with open(current_kpi_file) as f: + current_data = json.load(f) - # Create output directory if needed - output_file.parent.mkdir(parents=True, exist_ok=True) + if current_data.get("schema_version") != "2": + logger.error("Current KPI file must be schema_version 2 (hierarchical)") + return 1 - # Write analysis results to output file - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - f.write("\n") # Add EOL at EOF + current_records = _extract_kpi_records_from_hierarchical(current_data) + if not current_records: + logger.warning("No KPI records found in current file") + return 1 - logger.info(f"Analysis completed successfully, results written to: {output_file}") - return 0 + # Load baseline KPIs + baseline_kpi_data = find_baseline_kpis(historical_data_dir) + if not baseline_kpi_data: + _write_warning_report(output_file, current_kpi_file, plugin_module) + return 2 - except Exception as e: - logger.exception("KPI analysis failed") + # Build baseline index + baseline_index = _build_baseline_index(baseline_kpi_data, config) - # Create failure result - failure_result = { - "status": "failed", - "error": str(e), - "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, - "completed_at": time.time(), - } + # Run regression tests + results: list[KpiTestResult] = [] + skipped: list[dict[str, Any]] = [] - try: - # Attempt to write failure result - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(failure_result, f, indent=2) - f.write("\n") # Add EOL at EOF - except Exception: - logger.exception("Failed to write error result to output file") + for rec in current_records: + kpi_id = rec.get("kpi_id") + value = rec.get("value") + # Skip non-scalar values (2D KPIs, lists, etc.) + if not isinstance(value, (int, float)): + skipped.append({"kpi_id": kpi_id, "reason": "non-scalar value"}) + continue + + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + + baselines = baseline_index.get(key, []) + # Filter to only scalar baselines + scalar_baselines = [b for b in baselines if isinstance(b.get("value"), (int, float))] + + if len(scalar_baselines) < config.min_baseline_points: + skipped.append( + { + "kpi_id": kpi_id, + "labels": labels, + "reason": f"insufficient baselines ({len(scalar_baselines)} < {config.min_baseline_points})", + } + ) + continue + + result = _run_regression_test(rec, scalar_baselines, config) + results.append(result) + + # Sort results + results = _sort_results(results, config.sorting_keys) + + # Build report + baseline_sources = [str(p) for p in baseline_kpi_data.keys()] + report = _build_yaml_report( + results=results, + config=config, + current_source=str(current_kpi_file), + baseline_sources=baseline_sources, + skipped=skipped, + ) + + # Write YAML report + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + regressions = report["overall"]["regression_count"] + total = report["overall"]["total_tested"] + logger.info( + "Analysis complete: %d/%d KPIs tested, %d regressions", + total, + len(current_records), + regressions, + ) + + return 3 if regressions > 0 else 0 + + except Exception: + logger.exception("KPI analysis failed") return 1 +def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: + """Write a warning-level report when no baselines are available.""" + report = { + "analysis": { + "status": "NO_BASELINE", + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }, + "processed": { + "current_source": str(current_kpi_file), + "baseline_sources": [], + "baseline_source_count": 0, + }, + "tested": {"total_kpis": 0, "regressions": 0, "passes": 0, "skipped": 0}, + "results": [], + "overall": { + "verdict": "NO_BASELINE", + "message": "No historical KPI files found for regression testing", + "regression_count": 0, + "total_tested": 0, + "total_skipped": 0, + }, + } + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + def analyze_kpis( postprocess_config, # CaliperOrchestrationPostprocessConfig plugin_module: str, @@ -129,19 +415,15 @@ def analyze_kpis( if not postprocess_config.analyze.enabled: return {"status": "disabled", "reason": "analyze disabled"} - # Determine paths for analysis analyze_config = postprocess_config.analyze - # Historical KPIs directory path historical_kpis_dir = Path(analyze_config.historical_kpis) if not historical_kpis_dir.is_absolute(): historical_kpis_dir = output_dir / historical_kpis_dir - # Output path for analysis results output_path = output_dir / analyze_config.output try: - # Check if required files exist if not current_kpis_file.exists(): return { "status": "failed", @@ -156,7 +438,6 @@ def analyze_kpis( "completed_at": time.time(), } - # Call the core analysis function exit_code = run_kpi_analysis( current_kpi_file=current_kpis_file, historical_data_dir=historical_kpis_dir, @@ -165,123 +446,71 @@ def analyze_kpis( ) if exit_code == 0: - # Read the results from the output file - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return success with output file path for notifications - return { - "status": "success", - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": "Analysis completed but no output file was generated", - "completed_at": time.time(), - } + return { + "status": "success", + "output_file": str(output_path), + "completed_at": time.time(), + } elif exit_code == 2: - # Warning: no historical data found - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return warning status with message - return { - "status": "warning", - "message": result_data.get( - "message", "no historical KPI found for regression testing" - ), - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "baseline_files_count": 0, - "completed_at": time.time(), - } + return { + "status": "warning", + "message": "no historical KPI found for regression testing", + "output_file": str(output_path), + "completed_at": time.time(), + } + elif exit_code == 3: + return { + "status": "regression_detected", + "output_file": str(output_path), + "completed_at": time.time(), + } else: - # Read error from output file if it exists - error_msg = f"Analysis failed with exit code {exit_code}" - if output_path.exists(): - try: - with open(output_path) as f: - result_data = json.load(f) - error_msg = result_data.get("error", error_msg) - except Exception: - pass - return { "status": "failed", - "error": error_msg, + "error": f"Analysis failed with exit code {exit_code}", "completed_at": time.time(), } except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Analysis step failed with {error_msg}") - - # Don't suppress the exception - provide detailed error info and re-raise - import traceback - - full_traceback = traceback.format_exc() - logger.error(f"Full traceback for analyze_kpis failure:\n{full_traceback}") - - # Re-raise the original exception so the caller can see the full details - raise RuntimeError(f"KPI analysis failed: {error_msg}") from e + logger.exception("Analysis step failed") + raise RuntimeError(f"KPI analysis failed: {e}") from e def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: - """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. - - Args: - historical_dir: Directory to search for historical kpis.json files + """Load all kpis.json files from historical directory. - Returns: - Dictionary mapping file paths to loaded KPI JSON objects + Returns mapping of file paths to loaded hierarchical KPI data (schema v2 only). """ - baseline_kpis = {} + baseline_kpis: dict[Path, dict[str, Any]] = {} kpi_files = list(historical_dir.rglob("kpis.json")) if not kpi_files: - logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") + logger.warning("No kpis.json files found in: %s", historical_dir) return baseline_kpis - logger.info(f"Found {len(kpi_files)} historical KPI files to load") + logger.info("Found %d historical KPI files to load", len(kpi_files)) for kpi_file in kpi_files: try: with open(kpi_file) as f: kpi_data = json.load(f) - # Validate that it's a hierarchical format (schema_version 2) schema_version = kpi_data.get("schema_version", "unknown") if schema_version != "2": logger.warning( - f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" + "Skipping %s: unsupported schema version %s", kpi_file, schema_version ) continue baseline_kpis[kpi_file] = kpi_data - logger.debug(f"Loaded KPI file: {kpi_file}") + logger.debug("Loaded baseline: %s", kpi_file) except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON in {kpi_file}: {e}") - continue - except FileNotFoundError as e: - logger.error(f"KPI file not found: {kpi_file}: {e}") - continue + logger.error("Failed to parse JSON in %s: %s", kpi_file, e) except Exception as e: - logger.error(f"Failed to load KPI file {kpi_file}: {e}") - continue + logger.error("Failed to load %s: %s", kpi_file, e) - logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") + logger.info("Successfully loaded %d historical KPI files", len(baseline_kpis)) return baseline_kpis @@ -293,52 +522,33 @@ def run_analyze( plugin: Any = None, ) -> dict[str, Any]: """Run KPI analysis against ALL baseline files (CLI interface).""" - # For CLI use, we need to extract the baseline directory from the dict - # This is a bit of a hack, but the CLI interface expects a directory - if baseline_kpis: - # Get the parent directory of the first baseline file - first_baseline_path = next(iter(baseline_kpis.keys())) - historical_dir = first_baseline_path.parent - - # Extract plugin module name if available - plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=Path(current_path), - historical_data_dir=historical_dir, - output_file=Path(output_path), - plugin_module=plugin_module, - ) - - if exit_code == 0: - # Read results and convert to expected format for CLI - try: - with open(Path(output_path)) as f: - result_data = json.load(f) - return { - "status": "success", - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": result_data.get("completed_at", time.time()), - } - except Exception as e: - return { - "status": "failed", - "error": f"Failed to read analysis results: {e}", - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } - else: + if not baseline_kpis: return { "status": "failed", "error": "No baseline KPI files provided", "completed_at": time.time(), } + first_baseline_path = next(iter(baseline_kpis.keys())) + historical_dir = first_baseline_path.parent + + plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" + + exit_code = run_kpi_analysis( + current_kpi_file=Path(current_path), + historical_data_dir=historical_dir, + output_file=Path(output_path), + plugin_module=plugin_module, + ) + + if exit_code == 0: + return {"status": "success", "completed_at": time.time()} + elif exit_code == 2: + return {"status": "warning", "message": "no baselines found", "completed_at": time.time()} + elif exit_code == 3: + return {"status": "regression_detected", "completed_at": time.time()} + else: + return {"status": "failed", "error": f"exit code {exit_code}", "completed_at": time.time()} + # EOF diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py new file mode 100644 index 000000000..a71e01c3d --- /dev/null +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -0,0 +1,442 @@ +"""Tests for KPI regression analysis (analyze.py).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from projects.caliper.engine.kpi.analyze import ( + AnalysisConfig, + _build_baseline_index, + _extract_kpi_records_from_hierarchical, + _match_key, + _run_regression_test, + run_kpi_analysis, +) + + +def _make_hierarchical_kpi( + tests: list[dict], +) -> dict: + """Helper to build a schema_version=2 hierarchical KPI doc.""" + return {"schema_version": "2", "tests": tests} + + +def _make_test_entry( + run_id: str, + labels: dict, + kpis: list[dict], +) -> dict: + return { + "run_id": run_id, + "labels": labels, + "metadata": {"timestamp": "2025-01-01T00:00:00Z"}, + "kpis": kpis, + } + + +def _make_kpi(kpi_id: str, value, unit: str = "tokens/s", higher_is_better: bool = True) -> dict: + return {"id": kpi_id, "value": value, "unit": unit, "higher_is_better": higher_is_better} + + +class TestMatchKey: + def test_basic_match_key(self): + labels = {"platform": "A100", "version": "1.0", "os": "linux"} + key = _match_key(labels, ignored_keys=["os"], comparison_keys=["version"]) + assert ("os", "linux") not in key + assert ("version", "1.0") not in key + assert ("platform", "A100") in key + + def test_empty_config(self): + labels = {"a": "1", "b": "2"} + key = _match_key(labels, ignored_keys=[], comparison_keys=[]) + assert key == tuple(sorted([("a", "1"), ("b", "2")])) + + +class TestExtractRecords: + def test_extracts_flat_records(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("latency", 0.5, unit="s", higher_is_better=False), + ], + ), + ] + ) + records = _extract_kpi_records_from_hierarchical(data) + assert len(records) == 2 + assert records[0]["kpi_id"] == "throughput" + assert records[0]["labels"] == {"platform": "A100"} + assert records[1]["value"] == 0.5 + + def test_empty_tests(self): + data = _make_hierarchical_kpi([]) + assert _extract_kpi_records_from_hierarchical(data) == [] + + +class TestBuildBaselineIndex: + def test_indexes_by_kpi_and_match_key(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + _make_test_entry( + "run2", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + config = AnalysisConfig(comparison_keys=["version"]) + baseline_data = {Path("/fake/kpis.json"): data} + index = _build_baseline_index(baseline_data, config) + + mk = _match_key({"platform": "A100"}, [], ["version"]) + key = ("throughput", mk) + assert key in index + assert len(index[key]) == 2 + + +class TestRegressionTest: + def test_no_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 100.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 95.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + assert result.relative_change > 0 + + def test_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 80.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 100.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change < -0.1 + + def test_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 1.5, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}, {"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change > 0.1 + + def test_no_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 0.9, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + + +class TestEndToEnd: + """Full end-to-end test of run_kpi_analysis.""" + + def _write_hierarchical_kpi(self, path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(data, f) + + def test_pass_no_regression(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current KPIs: throughput=100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Historical: throughput=95, 98 (within 10% threshold) + baseline1 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + baseline2 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_2", + {"platform": "A100"}, + [ + _make_kpi("throughput", 98.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline1) + self._write_hierarchical_kpi(historical_dir / "run2" / "kpis.json", baseline2) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 0 + assert output_file.exists() + + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "PASS" + assert report["overall"]["verdict"] == "PASS" + assert report["overall"]["regression_count"] == 0 + assert report["tested"]["total_kpis"] == 1 + + def test_regression_detected(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: throughput dropped from 100 to 70 (30% regression) + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 70.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "REGRESSION_DETECTED" + assert report["overall"]["regression_count"] == 1 + assert report["results"][0]["verdict"] == "REGRESSION" + assert report["results"][0]["relative_change_pct"] == pytest.approx(-30.0) + + def test_no_historical_data(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + historical_dir.mkdir(parents=True) + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 2 + with open(output_file) as f: + report = yaml.safe_load(f) + assert report["overall"]["verdict"] == "NO_BASELINE" + + def test_mixed_regression_and_pass(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 2.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 1.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["overall"]["regression_count"] == 1 + latency_result = next(r for r in report["results"] if r["kpi_id"] == "latency") + assert latency_result["verdict"] == "REGRESSION" + throughput_result = next(r for r in report["results"] if r["kpi_id"] == "throughput") + assert throughput_result["verdict"] == "PASS" + + def test_skips_non_scalar_kpis(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("curve_data", [1.0, 2.0, 3.0]), # non-scalar + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert exit_code == 0 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["tested"]["skipped"] == 1 + assert report["tested"]["total_kpis"] == 1 + + def test_comparison_keys_separate_baselines(self, tmp_path): + """Records that differ on comparison_keys should still be matched.""" + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: version=2.0 on platform=A100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Baseline: version=1.0 on platform=A100 (different comparison key value) + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + # With version as comparison_key, both match on platform=A100 + # (plugin would expose this config; here we test the core logic directly) + exit_code = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + # Without plugin config, default AnalysisConfig has no comparison_keys, + # so version must also match → no baselines found → skip + assert exit_code == 0 or exit_code == 2 + with open(output_file) as f: + report = yaml.safe_load(f) + # The version label differs, so with empty comparison_keys they don't match + assert report["tested"]["total_kpis"] == 0 or report["overall"]["verdict"] == "NO_BASELINE" From d39659f6e4544778ca1b606a4e4d0d0767e11f64 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 11:21:52 +0200 Subject: [PATCH 0328/1072] [caliper] orchestration: postprocess: improve the step status reporting --- projects/caliper/orchestration/postprocess.py | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a32ef2736..3e3104c75 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1009,13 +1009,14 @@ def __init__( # State tracking self.steps: list[dict[str, Any]] = [] - self.parse_failed = False - self.visualize_failed = False - self.artifacts_to_kpis_failed = False - self.ai_data_failed = False - self.s3_import_failed = False - self.analyze_failed = False - self.s3_export_failed = False + # Initialize failure flags to None (skipped), will be set to False (success) or True (failure) + self.parse_failed = None + self.visualize_failed = None + self.artifacts_to_kpis_failed = None + self.ai_data_failed = None + self.s3_import_failed = None + self.analyze_failed = None + self.s3_export_failed = None # Configuration try: @@ -1147,7 +1148,7 @@ def _add_step( self.steps.append({step_name: step_data}) def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, Any]) -> bool: - """Check step result status and set appropriate failure flag if needed. + """Check step result status and set appropriate failure flag. Args: step_name: Name of the step @@ -1157,31 +1158,40 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A True if step failed or warned, False if successful """ status = result.get("status") - if status in ("failed", "warning"): - # Map step names to their failure flags - step_failure_map = { - "artifacts_to_kpis": "artifacts_to_kpis_failed", - "artifacts_to_ai_data": "ai_data_failed", - "s3_import": "s3_import_failed", - "analyse_kpis": "analyze_failed", - "s3_export": "s3_export_failed", - } - failure_attr = step_failure_map.get(step_name) - if failure_attr: - setattr(self, failure_attr, True) + # Map step names to their failure flags + step_failure_map = { + "artifacts_to_kpis": "artifacts_to_kpis_failed", + "artifacts_to_ai_data": "ai_data_failed", + "s3_import": "s3_import_failed", + "analyse_kpis": "analyze_failed", + "s3_export": "s3_export_failed", + } - if status == "warning": - warning_msg = result.get("message", "unknown warning") - logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") - elif status == "failed": - error_msg = result.get("error", "unknown error") - traceback_msg = result.get("traceback") - logger.error(f"Step '{step_name}' failed: {error_msg}") - if traceback_msg: - logger.error("Full traceback:\n%s", traceback_msg) + failure_attr = step_failure_map.get(step_name) + if not failure_attr: + return False + + if status in ("failed", "warning"): + setattr(self, failure_attr, True) + + if status == "warning": + warning_msg = result.get("message", "unknown warning") + logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") + elif status == "failed": + error_msg = result.get("error", "unknown error") + traceback_msg = result.get("traceback") + logger.error(f"Step '{step_name}' failed: {error_msg}") + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) return True + elif status == "success": + # Set flag to False for successful steps + setattr(self, failure_attr, False) + return False + + # For other statuses (disabled, etc.), leave flag as None return False def _get_step(self, step_name: str) -> dict[str, Any]: @@ -1263,6 +1273,7 @@ def _run_parse_step(self) -> None: ) if result.returncode == 0 and status_data and status_data.get("success", False): + self.parse_failed = False self._add_step( "parse", { @@ -1381,6 +1392,7 @@ def _run_visualize_step(self) -> None: # If output_dir is not under ARTIFACT_DIR, use absolute path relative_output_dir = str(output_dir) + self.visualize_failed = False self._add_step( "visualize", { @@ -1927,7 +1939,6 @@ def _compute_final_status(self) -> str: logger.info(f" s3_import_failed: {self.s3_import_failed}") logger.info(f" analyze_failed: {self.analyze_failed}") logger.info(f" s3_export_failed: {self.s3_export_failed}") - logger.info(f" analyze_failed: {self.analyze_failed}") final_status = compute_final_postprocess_status( test_outcome=self.test_outcome, From 2f6be00ab550fcd5df2bf6cbc6713d68d28fe0a6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 11:33:18 +0200 Subject: [PATCH 0329/1072] [caliper] orchestration: postprocess: better configure the analyze section --- projects/caliper/orchestration/postprocess.py | 28 ++++++------------- .../orchestration/postprocess_config.py | 19 +++++++++---- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 3e3104c75..939addacb 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1736,36 +1736,26 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: ) return - # Get KPI file path from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if not artifacts_to_kpis_step or artifacts_to_kpis_step.get("status") != "success": - self._add_step( - "analyse_kpis", - { - "status": "failed", - "error": "artifacts_to_kpis step did not complete successfully", - "completed_at": time.time(), - }, - ) - self.analyze_failed = True - return + # Get current KPI file path from config + current_kpis_file = Path(self.config.analyze.current_kpis) + if not current_kpis_file.is_absolute(): + current_kpis_path = output_dir / current_kpis_file + else: + current_kpis_path = current_kpis_file - current_kpis_file = artifacts_to_kpis_step.get("output_file") - if not current_kpis_file: + # Check if current KPI file exists + if not current_kpis_path.exists(): self._add_step( "analyse_kpis", { "status": "failed", - "error": "No output file found in artifacts_to_kpis step", + "error": f"Current KPI file not found: {current_kpis_path}", "completed_at": time.time(), }, ) self.analyze_failed = True return - # Convert relative path back to absolute path - current_kpis_path = env.ARTIFACT_DIR / current_kpis_file - result = _run_analyse_kpis( postprocess_config=self.config, output_dir=output_dir, diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index b30ccb222..13f789a8a 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -115,6 +115,10 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False + current_kpis: str | None = Field( + default="kpis.json", + description="Current KPI file path (relative to postprocess output dir unless absolute).", + ) historical_kpis: str | None = Field( default=None, description="Directory containing historical KPI JSON files (relative → postprocess output dir unless absolute).", @@ -125,11 +129,16 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): ) @model_validator(mode="after") - def _historical_kpis_when_enabled(self) -> Self: - if self.enabled and not (self.historical_kpis and str(self.historical_kpis).strip()): - raise ValueError( - "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." - ) + def _validate_when_enabled(self) -> Self: + if self.enabled: + if not (self.current_kpis and str(self.current_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty current_kpis file path." + ) + if not (self.historical_kpis and str(self.historical_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." + ) return self From 21fab2d9b5596fbbaaf5e2228095a8d1e61f040d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:05:04 +0200 Subject: [PATCH 0330/1072] [caliper] orchestration: postprocess: re-enable the hierarchical KPI format --- projects/caliper/orchestration/postprocess.py | 164 ------------------ 1 file changed, 164 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 939addacb..8fd31e2b3 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -323,170 +323,6 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() -def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: - """ - Transform flat KPI list into hierarchical JSON structure. - - Groups KPIs by test (run_id), extracting common labels and organizing - KPI metadata (name, help, unit, etc.) for improved readability. - - Args: - kpis: List of flat KPI records from compute_kpis - model: Unified model for accessing plugin metadata - - Returns: - Hierarchical JSON structure organized by test - """ - from collections import defaultdict - - if not kpis: - return {"schema_version": "2", "tests": []} - - # Group KPIs by test (run_id) - tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) - - # Get KPI function metadata from the plugin module - from projects.caliper.engine.kpi.decorators import get_kpi_functions - - try: - plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) - kpi_functions = get_kpi_functions(plugin_module_obj) - except (ImportError, AttributeError): - kpi_functions = {} - - # First pass: determine which labels are common (same value for all KPIs in a run) - # vs per-KPI (vary across KPIs, e.g. rate_index, intended_concurrency) - run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - for k, v in kpi.get("labels", {}).items(): - if k == "higher_is_better": - continue - run_label_values[run_id][k].add(str(v)) - - # Labels with more than one distinct value per run are per-KPI - per_kpi_label_keys: dict[str, set[str]] = {} - for run_id, label_vals in run_label_values.items(): - per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} - - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - test_data = tests_data[run_id] - varying_keys = per_kpi_label_keys.get(run_id, set()) - - kpi_labels = kpi.get("labels", {}) - test_labels = { - k: v - for k, v in kpi_labels.items() - if k not in ("higher_is_better",) and k not in varying_keys - } - - if not test_data["labels"]: - test_data["labels"] = test_labels - - # Store test metadata from first KPI - if not test_data["metadata"]: - test_data["metadata"] = { - "timestamp": kpi.get("timestamp"), - "source": kpi.get("source", {}), - "run_id": run_id, - } - - # Per-KPI labels that vary across rate points - kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} - - # Create KPI record with metadata - kpi_id = kpi.get("kpi_id") - raw_value = kpi.get("value") - - # Transform 2D KPI values to structured format - if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: - # Check if this looks like 2D data: list of tuples/lists with 2 elements - first_item = raw_value[0] - if isinstance(first_item, list | tuple) and len(first_item) == 2: - try: - # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format - structured_value = { - "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], - "count": len(raw_value), - } - final_value = structured_value - except (ValueError, TypeError, IndexError): - # If conversion fails, keep original value - final_value = raw_value - else: - final_value = raw_value - else: - final_value = raw_value - - kpi_record: dict[str, Any] = { - "id": kpi_id, - "value": final_value, - "unit": kpi.get("unit"), - "higher_is_better": kpi_labels.get("higher_is_better", True), - } - if kpi_specific_labels: - kpi_record["labels"] = kpi_specific_labels - - # Add KPI metadata from function decorator if available - if kpi_id in kpi_functions: - func = kpi_functions[kpi_id] - kpi_record.update( - { - "name": ( - func.__doc__.replace(" KPI.", "") - if func.__doc__ - else kpi_id.replace("_", " ").title() - ), - "help": getattr(func, "_kpi_help", ""), - } - ) - - # Add 2D-specific metadata if present - if getattr(func, "_kpi_is_2d", False): - kpi_record.update( - { - "is_2d": True, - "x_unit": getattr(func, "_kpi_x_unit", ""), - "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], - "y_help": getattr(func, "_kpi_y_help", None) - or getattr(func, "_kpi_help", ""), - } - ) - else: - kpi_record["is_2d"] = False - - # Add formatting info if available - if hasattr(func, "_kpi_format"): - kpi_record["format"] = func._kpi_format - else: - # Fallback if no function metadata available - kpi_record.update( - { - "name": kpi_id.replace("_", " ").title(), - "help": f"KPI: {kpi_id}", - "is_2d": isinstance(kpi.get("value"), list), - } - ) - - test_data["kpis"].append(kpi_record) - - # Convert to final structure - tests_list = [] - for run_id, test_data in tests_data.items(): - tests_list.append( - { - "run_id": run_id, - "labels": test_data["labels"], - "metadata": test_data["metadata"], - "kpis": test_data["kpis"], - } - ) - - return {"schema_version": "2", "tests": tests_list} - - def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, plugin, From 975112c5fba54a20585cef6f32ff56ff3da2b0db Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 14:27:21 +0200 Subject: [PATCH 0331/1072] [caliper] orchestration: postprocess: move logging code to postprocess_logging --- projects/caliper/orchestration/postprocess.py | 209 +----------------- .../orchestration/postprocess_logging.py | 165 ++++++++++++++ 2 files changed, 166 insertions(+), 208 deletions(-) create mode 100644 projects/caliper/orchestration/postprocess_logging.py diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 8fd31e2b3..a5a1f6661 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -11,7 +11,6 @@ import json import logging -import subprocess import time import traceback from pathlib import Path @@ -27,11 +26,11 @@ build_s3_export_command, build_s3_import_command, build_visualize_command, - log_caliper_start_banner, ) from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) +from projects.caliper.orchestration.postprocess_logging import _execute_caliper_command from projects.caliper.orchestration.postprocess_outcome import ( FINAL_KPI_PIPELINE_FAILED, FINAL_SUCCESS, @@ -46,212 +45,6 @@ logger = logging.getLogger(__name__) -def _format_command_for_log(command: list[str]) -> str: - """Format command for log file display with only --arguments on new lines. - - Args: - command: Command arguments list - - Returns: - Formatted command string for log file - """ - if not command: - return "" - - # Start with base command parts until we hit options - lines = [] - base_parts = [] - i = 0 - - # Collect all non-flag arguments into base_parts - while i < len(command) and not command[i].startswith("--"): - base_parts.append(command[i]) - i += 1 - - if base_parts: - lines.append(" ".join(base_parts)) - - # Handle --flag arguments on separate lines - while i < len(command): - if command[i].startswith("--"): - if i + 1 < len(command) and not command[i + 1].startswith("--"): - # Flag with value - lines.append(f"{command[i]} {command[i + 1]}") - i += 2 - else: - # Boolean flag - lines.append(command[i]) - i += 1 - else: - # Standalone argument - lines.append(command[i]) - i += 1 - - return " \\\n ".join(lines) - - -def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: - """Write step header to log file. - - Args: - log_file: Path to log file - command: Command that will be executed - step_name: Name of the step (e.g., "CALIPER PARSE") - """ - formatted_command = _format_command_for_log(command) - header = f"""{"=" * 60} -🚀 STARTING {step_name} (fork/exec) -📄 Command: - {formatted_command} -{"=" * 60} - -""" - with open(log_file, "w", encoding="utf-8") as f: - f.write(header) - - -def _write_step_footer_to_log_file( - log_file: Path, result: subprocess.CompletedProcess, step_name: str -) -> None: - """Write step footer to log file. - - Args: - log_file: Path to log file - result: Subprocess result - step_name: Name of the step (e.g., "CALIPER PARSE") - """ - footer = f""" - -{"=" * 60} -🏁 {step_name} COMPLETED -📊 Exit code: {result.returncode} -📄 Log file: {log_file} -{"=" * 60} -""" - with open(log_file, "a", encoding="utf-8") as f: - f.write(footer) - - -def _handle_caliper_output_and_completion_with_header( - result: subprocess.CompletedProcess, - log_file: Path, - status_file: Path, - step_name: str, -) -> dict: - """Handle Caliper output writing, status parsing, and log completion banner. - - This version appends to the log file instead of overwriting it, since we've - already written a header. - - Args: - result: Subprocess result from caliper execution - log_file: Path to write the step log file - status_file: Path to the status file to read and parse - step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") - - Returns: - Parsed status data dictionary - """ - import yaml - - # Append output to log file (don't overwrite since we've written header) - with open(log_file, "a", encoding="utf-8") as log_f: - if result.stdout: - log_f.write(result.stdout) - - # Also display output in main orchestration logs - if result.stdout: - prefix = step_name.upper() - for line in result.stdout.splitlines(): - logger.info("%s: %s", prefix, line) - - # Log banner indicating Caliper execution completed - logger.info("=" * 60) - logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) - logger.info("📊 Exit code: %s", result.returncode) - logger.info("📄 Log file: %s", log_file) - - # Read and parse status file - try: - with open(status_file, encoding="utf-8") as f: - status_data = yaml.safe_load(f) - # Handle case where YAML file is empty or invalid - if status_data is None: - status_data = { - "success": False, - "error": "Status file is empty or contains no valid YAML data", - } - except Exception as e: - status_data = {"success": False, "error": f"Failed to read status file: {e}"} - - # Log status file content for visibility - logger.info("📄 Status file content:") - if isinstance(status_data, dict): - for key, value in status_data.items(): - logger.info(" %s: %s", key, value) - else: - logger.info(" %s", status_data) - - logger.info("=" * 60) - - return status_data - - -def _execute_caliper_command( - command: list[str], - step_name: str, - status_file: Path, - step_logs_dir: Path, -) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: - """Execute a Caliper CLI command and return result and status data. - - Args: - command: CLI command arguments list - step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") - status_file: Path to read status YAML from - step_logs_dir: Directory to create log file in - - Returns: - Tuple of (subprocess result, parsed status data, log file path) - """ - # Create log file path with proper step index - from projects.caliper.orchestration.step_logging import _get_next_step_index - - step_index = _get_next_step_index(step_logs_dir) - step_name_safe = step_name.replace(" ", "_") - log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" - log_file.parent.mkdir(parents=True, exist_ok=True) - - # Create script file for debugging in step_scripts directory with index prefix - step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" - step_scripts_dir.mkdir(parents=True, exist_ok=True) - script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" - - # Log start banner and save script - log_caliper_start_banner(command, script_file, step_name.upper()) - - # Write header to log file - _write_step_header_to_log_file(log_file, command, step_name.upper()) - - # Execute command with combined stdout/stderr - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, # Combine stderr into stdout - text=True, - ) - - # Handle output, status parsing, and log completion banner - status_data = _handle_caliper_output_and_completion_with_header( - result, log_file, status_file, step_name.upper() - ) - - # Write footer to log file - _write_step_footer_to_log_file(log_file, result, step_name.upper()) - - return result, status_data, log_file - - def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. diff --git a/projects/caliper/orchestration/postprocess_logging.py b/projects/caliper/orchestration/postprocess_logging.py new file mode 100644 index 000000000..d4ea1704f --- /dev/null +++ b/projects/caliper/orchestration/postprocess_logging.py @@ -0,0 +1,165 @@ +""" +Command logging and execution utilities for Caliper postprocessing. + +Handles formatting commands for logs, writing log headers/footers, and executing +commands with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _format_command_for_log(command: list[str]) -> str: + """Format command for log file display with only --arguments on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for log file + """ + if not command: + return "" + + # Start with base command parts until we hit options + lines = [] + base_parts = [] + i = 0 + + # Collect all non-flag arguments into base_parts + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle --flag arguments on separate lines + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return " \\\n ".join(lines) + + +def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: + """Write step header to log file. + + Args: + log_file: Path to log file + command: Command that will be executed + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + formatted_command = _format_command_for_log(command) + header = f"""{"=" * 60} +🚀 STARTING {step_name} (fork/exec) +📄 Command: + {formatted_command} +{"=" * 60} + +""" + with open(log_file, "w", encoding="utf-8") as f: + f.write(header) + + +def _write_step_footer_to_log_file( + log_file: Path, result: subprocess.CompletedProcess, step_name: str +) -> None: + """Write step footer to log file. + + Args: + log_file: Path to log file + result: Subprocess result + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + footer = f""" + +{"=" * 60} +🏁 {step_name} COMPLETED +📊 Exit code: {result.returncode} +📄 Log file: {log_file} +{"=" * 60} +""" + with open(log_file, "a", encoding="utf-8") as f: + f.write(footer) + + +def _handle_caliper_output_and_completion_with_header( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + This version appends to the log file instead of overwriting it, since we've + already written a header. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Append output to log file (don't overwrite since we've written header) + with open(log_file, "a", encoding="utf-8") as log_f: + if result.stdout: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + if isinstance(status_data, dict): + for key, value in status_data.items(): + logger.info(" %s: %s", key, value) + else: + logger.info(" %s", status_data) + + logger.info("=" * 60) + + return status_data From b06798523256a2332a8c9255ccfd5f2ca3a2e872 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 16:38:03 +0200 Subject: [PATCH 0332/1072] [caliper] orchestration: postprocess: move code to caliper_invocation --- .../orchestration/caliper_invocation.py | 741 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 437 +++++------ .../orchestration/postprocess_logging.py | 7 + 3 files changed, 925 insertions(+), 260 deletions(-) create mode 100644 projects/caliper/orchestration/caliper_invocation.py diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py new file mode 100644 index 000000000..8acad1b07 --- /dev/null +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -0,0 +1,741 @@ +""" +Caliper CLI command invocation functions. + +This module contains functions that build and execute caliper CLI commands +using subprocess execution with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +import tempfile +import time +import traceback +from pathlib import Path +from typing import Any + +from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_analyse_kpis_command, + build_kpi_csv_export_command, + build_kpi_generate_command, + build_parse_command, + build_s3_export_command, + build_s3_import_command, + build_visualize_command, +) +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) +from projects.core.library import env + +logger = logging.getLogger(__name__) + + +def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: + """Convert absolute path to relative path from base directory. + + Args: + file_path: Absolute or relative file path + base_dir: Base directory to make path relative to + + Returns: + Relative path as string + """ + try: + path_obj = Path(file_path) + if path_obj.is_absolute() and base_dir: + base_obj = Path(base_dir) + if base_obj.is_absolute(): + try: + relative = path_obj.relative_to(base_obj) + return str(relative) + except ValueError: + # Path is not relative to base_dir, return as-is + return str(file_path) + return str(file_path) + except Exception: + return str(file_path) + + +def _execute_caliper_command( + command: list[str], + step_name: str, + status_file: Path, + step_logs_dir: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: + """Execute a Caliper CLI command and return result and status data. + + Args: + command: CLI command arguments list + step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") + status_file: Path to read status YAML from + step_logs_dir: Directory to create log file in + + Returns: + Tuple of (subprocess result, parsed status data, log file path) + """ + from projects.caliper.orchestration.cli_builder import log_caliper_start_banner + from projects.caliper.orchestration.postprocess_logging import ( + _handle_caliper_output_and_completion_with_header, + _write_step_footer_to_log_file, + _write_step_header_to_log_file, + ) + from projects.caliper.orchestration.step_logging import _get_next_step_index + + # Create log file path with proper step index + step_index = _get_next_step_index(step_logs_dir) + step_name_safe = step_name.replace(" ", "_") + log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Create script file for debugging in step_scripts directory with index prefix + step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" + step_scripts_dir.mkdir(parents=True, exist_ok=True) + script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" + + # Log start banner and save script + log_caliper_start_banner(command, script_file, step_name.upper()) + + # Write header to log file + _write_step_header_to_log_file(log_file, command, step_name.upper()) + + # Execute command with combined stdout/stderr + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = _handle_caliper_output_and_completion_with_header( + result, log_file, status_file, step_name.upper() + ) + + # Write footer to log file + _write_step_footer_to_log_file(log_file, result, step_name.upper()) + + return result, status_data, log_file + + +def run_artifacts_to_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Generate KPI JSON using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.enabled: + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } + if not postprocess_config.kpi.artifacts_to_kpis.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_kpis disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi generate", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_artifacts_to_ai_data( + postprocess_config, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export AI evaluation data using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.artifacts_to_ai_data.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_ai_data disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "ai_data_dir": relative_path, # Expected by postprocess.py + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"AI evaluation export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_kpis_to_csv( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export KPI JSON to CSV using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.csv.enabled: + return { + "status": "disabled", + "reason": "kpi.csv disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.csv.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" + + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + kpi_json_path=kpi_json_path, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return { + "status": "disabled", + "reason": "analyze disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("KPI analysis failed in run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_parse_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the parse step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.parse.enabled: + return True, {"status": "disabled", "reason": "parse disabled", "completed_at": time.time()} + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Parse completed successfully", + "test_directories": status_data.get("test_directories", []), + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper parse failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Parse step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during parse: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_visualize_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, + output_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the visualize step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.visualize.enabled: + return True, { + "status": "disabled", + "reason": "visualize disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_visualize_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Visualize completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper visualize failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Visualize step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during visualize: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_import_step( + config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the S3 import step. + + Returns: + Tuple of (success, step_result) + """ + if not config.s3.import_.enabled: + return True, { + "status": "disabled", + "reason": "s3_import disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_import_command( + config=config, + status_file=status_file, + output_dir=output_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-import", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "S3 import completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 import failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 import step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during s3-import: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_export_step( + config: CaliperOrchestrationPostprocessConfig, + step_logs_dir: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> dict[str, Any]: + """Execute the S3 export step. + + Returns: + Step result dictionary + """ + if not config.s3.export.enabled: + return { + "status": "disabled", + "reason": "s3_export disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_export_command( + config=config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return { + "status": "success", + "detail": "S3 export completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 export failed: %s", error_msg) + return { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 export step execution failed") + return { + "status": "failed", + "detail": f"Exception during s3-export: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a5a1f6661..70430ebd5 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -18,6 +18,10 @@ from pydantic import ValidationError +from projects.caliper.orchestration.caliper_invocation import ( + _execute_caliper_command, + run_analyse_kpis, +) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, build_kpi_csv_export_command, @@ -30,7 +34,6 @@ from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, ) -from projects.caliper.orchestration.postprocess_logging import _execute_caliper_command from projects.caliper.orchestration.postprocess_outcome import ( FINAL_KPI_PIPELINE_FAILED, FINAL_SUCCESS, @@ -115,11 +118,122 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() + # _transform_kpis_to_hierarchical_format function removed - CLI commands now handle format conversion + + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + test_data = tests_data[run_id] + + # Extract common labels (excluding KPI-specific ones) + kpi_labels = kpi.get("labels", {}) + test_labels = { + k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] + } # Exclude KPI-specific labels + + # Merge test labels (they should be the same for all KPIs in a test) + if not test_data["labels"]: + test_data["labels"] = test_labels + + # Store test metadata from first KPI + if not test_data["metadata"]: + test_data["metadata"] = { + "timestamp": kpi.get("timestamp"), + "source": kpi.get("source", {}), + "run_id": run_id, + } + + # Create KPI record with metadata + kpi_id = kpi.get("kpi_id") + raw_value = kpi.get("value") + + # Transform 2D KPI values to structured format + if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: + # Check if this looks like 2D data: list of tuples/lists with 2 elements + first_item = raw_value[0] + if isinstance(first_item, list | tuple) and len(first_item) == 2: + try: + # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format + structured_value = { + "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], + "count": len(raw_value), + } + final_value = structured_value + except (ValueError, TypeError, IndexError): + # If conversion fails, keep original value + final_value = raw_value + else: + final_value = raw_value + else: + final_value = raw_value + + kpi_record = { + "id": kpi_id, + "value": final_value, + "unit": kpi.get("unit"), + "higher_is_better": kpi_labels.get("higher_is_better", True), + } + + # Add KPI metadata from function decorator if available + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + kpi_record.update( + { + "name": ( + func.__doc__.replace(" KPI.", "") + if func.__doc__ + else kpi_id.replace("_", " ").title() + ), + "help": getattr(func, "_kpi_help", ""), + } + ) + + # Add 2D-specific metadata if present + if getattr(func, "_kpi_is_2d", False): + kpi_record.update( + { + "is_2d": True, + "x_unit": getattr(func, "_kpi_x_unit", ""), + "x_help": getattr(func, "_kpi_x_help", ""), + "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_help": getattr(func, "_kpi_y_help", None) + or getattr(func, "_kpi_help", ""), + } + ) + else: + kpi_record["is_2d"] = False + + # Add formatting info if available + if hasattr(func, "_kpi_format"): + kpi_record["format"] = func._kpi_format + else: + # Fallback if no function metadata available + kpi_record.update( + { + "name": kpi_id.replace("_", " ").title(), + "help": f"KPI: {kpi_id}", + "is_2d": isinstance(kpi.get("value"), list), + } + ) + + test_data["kpis"].append(kpi_record) + + # Convert to final structure + tests_list = [] + for run_id, test_data in tests_data.items(): + tests_list.append( + { + "run_id": run_id, + "labels": test_data["labels"], + "metadata": test_data["metadata"], + "kpis": test_data["kpis"], + } + ) + + return {"schema_version": "2", "tests": tests_list} + def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, @@ -181,43 +295,6 @@ def _run_artifacts_to_kpis( # Convert to expected format if status_data.get("success"): - # Transform JSONL (schema v1) to hierarchical JSON (schema v2) - try: - logger.info(f"Transforming KPI output to hierarchical format: {output_file}") - - # Read the generated JSONL file - kpis = [] - with open(output_file, encoding="utf-8") as f: - for line in f: - line = line.strip() - if line: - import json - - kpis.append(json.loads(line)) - - if kpis: - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) - - import json - - with open(output_file, "w", encoding="utf-8") as f: - json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) - - logger.info( - f"Successfully transformed {len(kpis)} KPI records to hierarchical format" - ) - else: - logger.warning("No KPI records found in output file") - - except Exception as transform_error: - logger.error(f"Failed to transform KPIs to hierarchical format: {transform_error}") - return { - "status": "failed", - "error": f"KPI transformation failed: {transform_error}", - "completed_at": time.time(), - "log_file": log_file, - } - relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) logger.info( f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" @@ -251,8 +328,6 @@ def _run_artifacts_to_kpis( def _run_artifacts_to_ai_data( postprocess_config, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, @@ -346,94 +421,8 @@ def _load_test_labels(test_dir: Path) -> dict[str, Any]: return {} -def _export_test_entries_with_artifacts( - model, ai_data_dir: Path, base_dir: Path, plugin -) -> list[dict]: - """ - Export test entries by creating directories and copying specific artifacts. - - Args: - model: Unified model containing test results - ai_data_dir: Directory where test entries should be exported - base_dir: Base directory of the test artifacts (test directory) - plugin: Plugin instance to get artifact file list - - Returns: - List of exported test entry information - """ - import shutil - - exported_entries = [] - - for idx, record in enumerate(model.unified_result_records): - # Create directory for this test entry - test_entry_dir = ai_data_dir / f"test_entry_{idx:03d}" - test_entry_dir.mkdir(parents=True, exist_ok=True) - - # Load test labels from __test_labels__.yaml if available - test_dir = base_dir / record.test_base_path - test_labels = _load_test_labels(test_dir) - - # Record test entry metadata - entry_info = { - "entry_id": f"test_entry_{idx:03d}", - "test_base_path": str(record.test_base_path), - "distinguishing_labels": record.distinguishing_labels, - "test_labels": test_labels, - "copied_files": [], - "missing_files": [], - } - - # Get artifact files specific to this test directory only (plugin is scoped to test directory) - relevant_files = plugin.get_ai_data_artifact_files_for_test(test_dir) - - logger.debug( - f"Test entry {idx}: found {len(relevant_files)} artifact files in test directory {test_dir}" - ) - - # Copy relevant files for this test entry (preserving directory structure) - for target_file in relevant_files: - # source is test_dir + test_relative_path, target is test_entry_dir + test_relative_path - source_file = test_dir / target_file - target_path = test_entry_dir / target_file - - if source_file.exists(): - target_path.parent.mkdir(parents=True, exist_ok=True) - - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "relative_path": target_file, - "size_bytes": source_file.stat().st_size, - } - ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) - else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) - - # Write entry metadata - entry_metadata_file = test_entry_dir / "entry_metadata.json" - - with open(entry_metadata_file, "w") as f: - json.dump(entry_info, f, indent=2) - - exported_entries.append(entry_info) - - return exported_entries - - def _run_kpis_to_csv( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, kpi_json_path: Path, base_dir: Path, @@ -552,35 +541,23 @@ def _run_analyse_kpis( # Create temporary status file for subprocess communication status_file = output_dir / "analyse_kpis_status.yaml" - # Note: CLI command building removed in favor of direct function call - - # Load plugin for KPI definitions - from projects.caliper.engine.load_plugin import load_plugin - - plugin = load_plugin(plugin_module) - - # Find the most recent baseline file - from projects.caliper.engine.kpi.analyze_hierarchical import ( - analyze_hierarchical_kpis, - find_most_recent_baseline, + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, ) - baseline_file = find_most_recent_baseline(historical_kpis_dir) - if not baseline_file: - return { - "status": "failed", - "error": f"No baseline KPI files found in {historical_kpis_dir}", - "completed_at": time.time(), - "log_file": None, - } - - # Run hierarchical KPI analysis using core engine - logger.info(f"Running KPI analysis: {current_kpis_file} vs {baseline_file}") - result = analyze_hierarchical_kpis( - current_kpis_path=current_kpis_file, - baseline_kpis_path=baseline_file, - output_path=output_file, - plugin=plugin, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, ) # Clean up temporary status file @@ -589,19 +566,16 @@ def _run_analyse_kpis( except FileNotFoundError: pass - # Handle the analysis result - if result["status"] == "success": - logger.info( - f"Analysis completed: {result['regressions_count']} regressions, {result['improvements_count']} improvements" - ) + # Convert to expected format + if status_data.get("success"): return { "status": "success", "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), - "findings_count": result["findings_count"], - "regressions_count": result["regressions_count"], - "improvements_count": result["improvements_count"], + "findings_count": status_data.get("findings_count", 0), + "regressions_count": status_data.get("regressions_count", 0), + "improvements_count": status_data.get("improvements_count", 0), "completed_at": time.time(), - "log_file": None, + "log_file": log_file, } else: return { @@ -1077,117 +1051,66 @@ def _run_kpi_and_ai_data_steps(self) -> None: if not self.config.kpi.enabled: return + # Setup output directory and module string with focused error handling try: # Determine output directory for KPI/AI data steps - use base artifact directory output_dir = env.ARTIFACT_DIR logger.info(f"KPI steps using base artifact directory: {output_dir}") output_dir.mkdir(parents=True, exist_ok=True) - # Load plugin and model - from projects.caliper.engine.load_plugin import load_plugin - from projects.caliper.engine.parse import run_parse - from projects.caliper.engine.plugin_config import resolve_plugin_module_string - - # Load plugin - mod_str, _manifest = resolve_plugin_module_string( - base_dir=self.tree_root, - postprocess_config=self.manifest_path, - cli_plugin=self.config.plugin_module, - ) - plugin = load_plugin(mod_str) - - # Parse model - model = run_parse( - base_dir=self.tree_root, - plugin_module=mod_str, - plugin=plugin, - use_cache=not self.config.parse.no_cache, - ) - - # KPI JSON generation - self._run_artifacts_to_kpis_step(plugin, model, output_dir, mod_str) - - # KPI CSV export - self._run_kpis_to_csv_step(plugin, model, output_dir) - - # AI evaluation export - self._run_artifacts_to_ai_data_step(plugin, model, output_dir, mod_str) - - # S3 import (historical data) - self._run_s3_import_step(output_dir) - - # Analyze KPIs (current vs historical) - moved before S3 export - self._run_analyse_kpis_step(output_dir, mod_str) - - # S3 export - self._run_s3_export_step(output_dir) - + # Resolve plugin module string (this is just string manipulation, not engine access) + mod_str = self.config.plugin_module or "unknown" except Exception as e: completion_time = time.time() - logger.error(f"Failed to run KPI/AI eval operations: {e}") - self._add_step( + logger.error(f"Failed to setup KPI/AI eval operations: {e}") + # Only mark all steps as failed if basic setup fails + for step_name in [ "artifacts_to_kpis", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "kpis_to_csv", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "artifacts_to_ai_data", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "s3_import", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "analyse_kpis", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "s3_export", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) + ]: + self._add_step( + step_name, + { + "status": "failed", + "error": f"Setup failed: {e}", + "completed_at": completion_time, + }, + ) self.artifacts_to_kpis_failed = True self.ai_data_failed = True self.s3_import_failed = True self.analyze_failed = True self.s3_export_failed = True + return - def _run_artifacts_to_kpis_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + # Run each step independently - each has its own error handling + # KPI JSON generation + self._run_artifacts_to_kpis_step(output_dir, mod_str) + + # KPI CSV export + self._run_kpis_to_csv_step(output_dir) + + # AI evaluation export + self._run_artifacts_to_ai_data_step(output_dir, mod_str) + + # S3 import (historical data) + self._run_s3_import_step(output_dir) + + # Analyze KPIs (current vs historical) - moved before S3 export + self._run_analyse_kpis_step(output_dir, mod_str) + + # S3 export + self._run_s3_export_step(output_dir) + + def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: result = _run_artifacts_to_kpis( self.config, - plugin, - model, output_dir, mod_str, self.tree_root, @@ -1207,7 +1130,7 @@ def _run_artifacts_to_kpis_step( }, ) - def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> None: + def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" if not self.config.kpi.kpis_to_csv.enabled: self._add_step( @@ -1223,8 +1146,6 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output result = _run_kpis_to_csv( self.config, - plugin, - model, output_dir, kpi_json_path, self.tree_root, @@ -1237,9 +1158,7 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No # CSV export failure doesn't affect overall status - it's supplementary logger.warning("KPI CSV export failed but continuing execution") - def _run_artifacts_to_ai_data_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + def _run_artifacts_to_ai_data_step(self, output_dir: Path, mod_str: str) -> None: """Execute the AI evaluation export step.""" if not self.config.kpi.artifacts_to_ai_data.enabled: self._add_step( @@ -1255,8 +1174,6 @@ def _run_artifacts_to_ai_data_step( try: result = _run_artifacts_to_ai_data( self.config, - plugin, - model, output_dir, mod_str, self.tree_root, @@ -1385,7 +1302,7 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: self.analyze_failed = True return - result = _run_analyse_kpis( + result = run_analyse_kpis( postprocess_config=self.config, output_dir=output_dir, plugin_module=plugin_module, diff --git a/projects/caliper/orchestration/postprocess_logging.py b/projects/caliper/orchestration/postprocess_logging.py index d4ea1704f..fd4e15daa 100644 --- a/projects/caliper/orchestration/postprocess_logging.py +++ b/projects/caliper/orchestration/postprocess_logging.py @@ -149,6 +149,13 @@ def _handle_caliper_output_and_completion_with_header( "success": False, "error": "Status file is empty or contains no valid YAML data", } + # Normalize non-mapping payloads (lists, strings, integers) into failure dict + elif not isinstance(status_data, dict): + status_data = { + "success": False, + "error": f"Status file contains non-mapping data: {type(status_data).__name__}", + "raw_data": status_data, + } except Exception as e: status_data = {"success": False, "error": f"Failed to read status file: {e}"} From 0a43c5da9fe48e9c51b6d8c2683d07ace1e97339 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:29:09 +0200 Subject: [PATCH 0333/1072] [llm_d] orchestration: test_phase: save extra KPI labels --- projects/llm_d/orchestration/test_phase.py | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 7ba815fbc..6e8455b2b 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -140,6 +140,32 @@ def ensure_kueue_local_queue() -> None: logger.info("LocalQueue %s created successfully", queue_name) +def extract_kpi_labels_from_config() -> dict[str, str]: + """Extract kpi_labels from project configuration. + + Returns: + Dictionary of kpi_labels for system context + """ + kpi_labels = {} + + # Extract platform name + platform_name = config.project.get_config("cpt.kpi.labels.platform_name") + if platform_name: + kpi_labels["platform"] = platform_name + + # Extract gpu_type + gpu_type = config.project.get_config("cpt.kpi.labels.gpu_type") + if gpu_type: + kpi_labels["gpu_type"] = gpu_type + + # Extract test_harness + test_harness = config.project.get_config("cpt.kpi.labels.test_harness") + if test_harness: + kpi_labels["test_harness"] = test_harness + + return kpi_labels + + def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" @@ -155,7 +181,10 @@ def create_test_labels() -> None: if benchmark_keys: labels["guidellm_loadshape"] = benchmark_keys[0] - write_test_labels(env.ARTIFACT_DIR, labels) + # Extract kpi_labels from config + kpi_labels = extract_kpi_labels_from_config() + + write_test_labels(env.ARTIFACT_DIR, labels, kpi_labels=kpi_labels if kpi_labels else None) logger.info("Created test labels: %s", labels) # Dump config.project to config.yaml From a2c5a8642f01ff13287c21e909656060d8b70665 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:29:53 +0200 Subject: [PATCH 0334/1072] [core] library: postprocess: allow saving kpi_labels in the test_labels file --- projects/core/library/postprocess.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 5fe2c100a..8eccb5af4 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -33,7 +33,12 @@ def write_test_labels( - directory: Path, labels: dict[str, str], *, version: str = "1", dump_config: bool = True + directory: Path, + labels: dict[str, str], + *, + version: str = "1", + dump_config: bool = True, + kpi_labels: dict[str, str] | None = None, ) -> Path: """Write a __test_labels__.yaml file to mark a directory as a Caliper test base. @@ -42,6 +47,7 @@ def write_test_labels( labels: Dictionary of label key-value pairs version: Version string for the test labels format (default: "1") dump_config: Whether to save project configuration to config.yaml (default: True) + kpi_labels: Optional dictionary of KPI labels for system context Returns: Path to the created __test_labels__.yaml file @@ -53,6 +59,10 @@ def write_test_labels( "model": "llama-3", "deployment": "single-zone", "rate": "10" + }, + kpi_labels={ + "platform": "CKS", + "gpu_type": "H100" } ) """ @@ -62,6 +72,9 @@ def write_test_labels( "labels": labels, } + if kpi_labels: + payload["kpi_labels"] = kpi_labels + # Create directory and write YAML test_labels_path.parent.mkdir(parents=True, exist_ok=True) with test_labels_path.open("w", encoding="utf-8") as handle: From 6201c537897c5f570166090454497790f453f7ae Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:31:13 +0200 Subject: [PATCH 0335/1072] [guidellm] postprocess: guidellm: generate the KPI labels --- .../postprocess/guidellm/parsing/kpis.py | 47 +++-- .../postprocess/guidellm/parsing/parsers.py | 190 ++++++++++++++++++ 2 files changed, 219 insertions(+), 18 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index 2eb4ac959..35ecf5a85 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -13,7 +13,6 @@ LowerBetter, TwoDimensional, build_catalog_from_functions, - create_label_extractor, get_kpi_functions, is_2d_kpi, ) @@ -220,13 +219,34 @@ def guidellm_tpot_p99(unified_record) -> list[tuple[float, float]]: class GuideLLMKpiHandler: """Handles KPI catalog and computation for GuideLLM benchmarks.""" - # Define label extractor for all GuideLLM test conditions - LABEL_EXTRACTOR = create_label_extractor( - { - "strategy": "metrics.strategy", - "duration": "metrics.duration", + # Define custom label extractor for GuideLLM with fallbacks for missing kpi_labels + @staticmethod + def _extract_labels(record) -> dict[str, Any]: + """Extract labels from record with fallbacks for missing kpi_labels.""" + + from projects.caliper.engine.kpi.decorators import _extract_value_by_path + + labels = {} + + # Extract artifact-based labels (graceful handling) + artifact_fields = { + "product_version": "metrics.product_version", + "cluster": "metrics.cluster", + "deployment_profile": "metrics.deployment_profile", + "model_name": "metrics.model_name", + "load_shape": "metrics.benchmark_key", } - ) + + for label_key, path in artifact_fields.items(): + value = _extract_value_by_path(record, path) + if value is not None: + labels[label_key] = str(value) + + labels |= record.metrics.get("kpi_labels", {}) or {} + + return labels + + LABEL_EXTRACTOR = type("TestLabelExtractor", (), {"extract": _extract_labels})() # Metadata fields to include in KPI records but not as labels @staticmethod @@ -284,7 +304,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: # Generate scalar KPIs for each record for r in valid_records: - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -303,9 +322,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if value is None: continue - # Merge base labels, test condition labels, and system labels + # Use only extracted KPI labels, not test labels all_labels = { - **base_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } @@ -338,7 +356,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not curves or not request_rates: continue - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -357,14 +374,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not value or value is None: continue - # Remove rate-specific labels for aggregated KPIs - aggregated_labels = { - k: v - for k, v in base_labels.items() - if k not in ["concurrency", "rate", "max_concurrency"] - } + # Use only extracted KPI labels, not test labels all_labels = { - **aggregated_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index 4208b2bcf..3853ef99f 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -7,6 +7,9 @@ from pathlib import Path from typing import Any +import jsonpath_ng +import yaml + from projects.caliper.engine.model import ( ParseResult, TestBaseNode, @@ -27,6 +30,77 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: return {"facet": "default"} +def _kpi_labels_from_node(node: TestBaseNode) -> dict[str, Any]: + """Extract kpi_labels from a test node.""" + raw = node.labels + kpi_labels = raw.get("kpi_labels") + if isinstance(kpi_labels, dict): + return dict(kpi_labels) + return {} + + +def extract_field_by_jsonpath(data: dict[str, Any], jsonpath: str, default: Any = None) -> Any: + """ + Extract a field from nested dictionary using JSONPath notation. + + Args: + data: The dictionary to extract from + jsonpath: JSONPath like "spec.model.name" or "spec.template.containers[0].name" + default: Default value if field not found + + Returns: + The extracted value or default if not found + + Examples: + extract_field_by_jsonpath(yaml_data, "spec.model.name") + extract_field_by_jsonpath(yaml_data, "spec.template.containers[0].name") + """ + try: + return jsonpath_ng.parse(jsonpath).find(data)[0].value + except IndexError: + return default + + +def parse_product_version_from_annotation(annotation_value: str) -> str | None: + """ + Parse product version from kserve config annotation. + + Args: + annotation_value: Raw annotation like "v3-5-0-ea-2-kserve-config-llm-decode-template" + + Returns: + Cleaned version like "v3.5.0-ea.2" or None if parsing fails + + Examples: + parse_product_version_from_annotation("v3-5-0-ea-2-kserve-config-llm-decode-template") + # Returns: "v3.5.0-ea.2" + """ + if not annotation_value: + return None + + # Remove the kserve suffix + suffix = "-kserve-config-llm-decode-template" + if annotation_value.endswith(suffix): + version_part = annotation_value[: -len(suffix)] + else: + version_part = annotation_value + + # Transform v3-5-0-ea-2 -> v3.5.0-ea.2 + # Replace hyphens with dots in the version numbers, keep -ea- as is + if version_part.startswith("v") and "-ea-" in version_part: + # Split on -ea- to handle the pre-release part separately + base_version, ea_part = version_part.split("-ea-", 1) + + # Replace hyphens with dots in base version (v3-5-0 -> v3.5.0) + base_version = base_version.replace("-", ".") + + # Reconstruct with -ea. format + cleaned_version = f"{base_version}-ea.{ea_part}" + return cleaned_version + + return None + + class GuideLLMParser: """Parser for GuideLLM benchmark JSON artifacts.""" @@ -36,6 +110,99 @@ def _is_benchmarks_artifact(path: Path) -> bool: path.name.startswith("benchmarks-rate-") and path.suffix == ".json" ) + @staticmethod + def _is_llmisvc_artifact(path: Path) -> bool: + """Check if path is an LLMInferenceService YAML artifact.""" + return path.name in [ + "llminferenceservice.yaml", + "llminferenceservice.yml", + ] and "__capture_llmisvc_state" in str(path) + + @staticmethod + def _is_config_artifact(path: Path) -> bool: + """Check if path is a config.yaml artifact.""" + return path.name == "config.yaml" + + def extract_fields_from_llmisvc(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from LLMInferenceService YAML file. + + Args: + file_path: Path to llminferenceservice.yaml file + + Returns: + Dictionary with extracted fields (product_version, deployment_profile, model_name) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract product version from kserve annotation + annotation_value = extract_field_by_jsonpath( + yaml_data, 'status.annotations["serving.kserve.io/config-llm-decode-template"]' + ) + if annotation_value: + product_version = parse_product_version_from_annotation(annotation_value) + if product_version: + result["product_version"] = product_version + logging.info(f"Extracted product_version '{product_version}' from {file_path}") + + # Extract deployment profile from forge annotation + deployment_profile = extract_field_by_jsonpath( + yaml_data, 'metadata.annotations["forge.openshift.io/deployment-profile"]' + ) + if deployment_profile: + result["deployment_profile"] = deployment_profile + logging.info( + f"Extracted deployment_profile '{deployment_profile}' from {file_path}" + ) + + # Extract model name from spec + model_name = extract_field_by_jsonpath(yaml_data, "spec.model.name") + if model_name: + result["model_name"] = model_name + logging.info(f"Extracted model_name '{model_name}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + + def extract_fields_from_config(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from config.yaml file. + + Args: + file_path: Path to config.yaml file + + Returns: + Dictionary with extracted fields (cluster, benchmark_key) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract cluster name + cluster = extract_field_by_jsonpath(yaml_data, "ci_job.cluster") + if cluster: + result["cluster"] = cluster + logging.info(f"Extracted cluster '{cluster}' from {file_path}") + + # Extract benchmark key + benchmark_key = extract_field_by_jsonpath(yaml_data, "runtime.benchmark_key") + if benchmark_key: + result["benchmark_key"] = benchmark_key + logging.info(f"Extracted benchmark_key '{benchmark_key}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + def parse_benchmarks_json( self, file_path: Path ) -> tuple[list[GuideLLMBenchmark], GuideLLMConfiguration | None, list[str]]: @@ -376,6 +543,10 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: benchmarks_files = [p for p in node.artifact_paths if self._is_benchmarks_artifact(p)] benchmarks_files.sort(key=lambda path: path.name) + # Also look for LLMInferenceService YAML files and config.yaml files for system information + llmisvc_files = [p for p in node.artifact_paths if self._is_llmisvc_artifact(p)] + config_files = [p for p in node.artifact_paths if self._is_config_artifact(p)] + if not benchmarks_files: # No benchmark result JSON found for this node, create empty record labels = _labels_from_node(node) @@ -408,6 +579,25 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: if node_config: metrics["configuration"] = node_config.to_dict() + # Extract fields from LLMInferenceService YAML if available + for llmisvc_file in llmisvc_files: + llmisvc_fields = self.extract_fields_from_llmisvc(llmisvc_file) + for field_name, field_value in llmisvc_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract fields from config.yaml if available + for config_file in config_files: + config_fields = self.extract_fields_from_config(config_file) + for field_name, field_value in config_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract kpi_labels from test labels + kpi_labels = _kpi_labels_from_node(node) + if kpi_labels: + metrics["kpi_labels"] = kpi_labels + records.append( UnifiedResultRecord( test_base_path=str(node.test_path), From 1cafc9eabf3f1391da711adc0eef348715368524 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:33:30 +0200 Subject: [PATCH 0336/1072] [llm_d] orchestration: config: xks CPT labels --- projects/llm_d/orchestration/config.d/cpt.yaml | 5 +++++ projects/llm_d/orchestration/presets.d/cks.yaml | 4 ++++ projects/llm_d/orchestration/presets.d/janus.yaml | 4 ++++ 3 files changed, 13 insertions(+) create mode 100644 projects/llm_d/orchestration/config.d/cpt.yaml diff --git a/projects/llm_d/orchestration/config.d/cpt.yaml b/projects/llm_d/orchestration/config.d/cpt.yaml new file mode 100644 index 000000000..bfb9b06c8 --- /dev/null +++ b/projects/llm_d/orchestration/config.d/cpt.yaml @@ -0,0 +1,5 @@ +kpi: + labels: + platform_name: null + gpu_type: null + test_harness: null diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index 1b59f614b..b03451fce 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -17,6 +17,10 @@ cks: platform.inference_service.scheduler.node_selector: gpu.nvidia.com/class: H200 + cpt.kpi.labels.platform_name: CKS + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + cks-intelligent: extends: - cks diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml index 9612228ce..ed629be57 100644 --- a/projects/llm_d/orchestration/presets.d/janus.yaml +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -10,6 +10,10 @@ janus: platform.gateway.status_address_name: null platform.gateway.status_address_name: gateway-internal + cpt.kpi.labels.platform_name: OCP + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + janus-intelligent: extends: - janus From 60b9c09c8be027312a42dd60bbc68fbe774d8282 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 17:56:19 +0200 Subject: [PATCH 0337/1072] [caliper] KPI analyze --- projects/caliper/cli/commands.py | 278 +++++------- projects/caliper/cli/kpi_analysis.py | 27 +- projects/caliper/cli/main.py | 4 +- projects/caliper/engine/kpi/analyze.py | 425 ++++++++++++++---- .../engine/kpi/analyze_hierarchical.py | 153 +++++-- projects/caliper/engine/kpi/format.py | 1 + projects/caliper/engine/kpi/generate.py | 22 +- projects/caliper/orchestration/postprocess.py | 112 +---- 8 files changed, 610 insertions(+), 412 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 4ae1710be..22be577ec 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -13,7 +13,6 @@ from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml -from projects.caliper.engine.kpi.analyze import run_analyze from projects.caliper.engine.kpi.generate import run_kpi_generate from projects.caliper.engine.kpi.import_export import ( import_kpis_snapshot, @@ -447,6 +446,13 @@ def ai_eval_export( @click.command("generate") @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option( + "--format", + "format_type", + type=click.Choice(["hierarchical", "jsonl"], case_sensitive=False), + default="hierarchical", + help="Output format: hierarchical JSON (default) or JSONL", +) @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) @@ -454,6 +460,7 @@ def ai_eval_export( def kpi_generate( ctx: click.Context, output: Path, + format_type: str, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -478,6 +485,7 @@ def kpi_generate( output=output, use_cache=True, cache_path=None, + format_type=format_type, ) status_data = {"success": True, "output_file": str(output)} click.echo(f"Generated {output}") @@ -602,97 +610,6 @@ def kpi_import(ctx: click.Context, snapshot: Path) -> None: click.echo(f"Imported {snapshot}") -@click.command("analyze") -@click.option( - "--current", type=click.Path(path_type=Path), required=True, help="Current KPI file to analyze" -) -@click.option( - "--baseline-dir", - type=click.Path(path_type=Path), - required=True, - help="Directory containing historical KPI files (will use most recent)", -) -@click.option( - "--output", - type=click.Path(path_type=Path), - required=True, - help="Output file for analysis results", -) -@click.option( - "--plugin", - "plugin_module", - metavar="MODULE", - required=True, - help="Plugin module for KPI definitions and analysis rules", -) -@click.pass_context -def kpi_analyze( - ctx: click.Context, - current: Path, - baseline_dir: Path, - output: Path, - plugin_module: str, -) -> None: - """Analyze KPIs for regressions using hierarchical format.""" - try: - from projects.caliper.engine.kpi.analyze_hierarchical import find_most_recent_baseline - from projects.caliper.engine.load_plugin import load_plugin - - # Load plugin for KPI definitions and analysis rules - plugin = load_plugin(plugin_module) - if not plugin: - click.echo(f"❌ Failed to load plugin: {plugin_module}", err=True) - sys.exit(1) - - click.echo(f"🔌 Using plugin: {plugin_module}") - - baseline_kpis = find_most_recent_baseline(baseline_dir) - if not baseline_kpis: - click.echo( - f"❌ No kpis.json files found in baseline directory: {baseline_dir}", err=True - ) - sys.exit(1) - - click.echo(f"📊 Found {len(baseline_kpis)} baseline files to process") - - # Run analysis with ALL baseline files (not just the most recent) - result = run_analyze( - current_path=current, baseline_kpis=baseline_kpis, output_path=output, plugin=plugin - ) - - # Check result status at CLI level - if result["status"] == "success": - click.echo(f"✅ Analysis complete. Results written to: {output}") - elif result["status"] == "skipped": - click.echo(f"⚠️ Analysis skipped: {result.get('reason', 'Unknown reason')}") - click.echo(f"📝 Output written to: {output}") - else: - click.echo(f"❌ Analysis failed: {result.get('error', 'Unknown error')}", err=True) - sys.exit(1) - - # Show the analysis results on screen - if output.exists(): - try: - import json - - with open(output) as f: - result_data = json.load(f) - - # Convert to YAML for better readability - yaml_output = yaml.dump(result_data, default_flow_style=False, indent=2) - click.echo("\n📊 Analysis Results:") - click.echo("=" * 50) - click.echo(yaml_output) - except Exception as e: - click.echo(f"⚠️ Could not display results: {e}") - else: - click.echo("⚠️ Output file not found") - except Exception as e: # noqa: BLE001 - raise e - click.echo(f"❌ kpi analyze failed: {e}", err=True) - sys.exit(3) - - @click.command("analyse-kpis") @click.option( "--output", @@ -702,13 +619,13 @@ def kpi_analyze( ) @click.option( "--current-kpis-file", - type=click.Path(path_type=Path, exists=True), + type=click.Path(path_type=Path), required=True, help="Path to current KPIs JSON file", ) @click.option( "--historical-kpis-dir", - type=click.Path(path_type=Path, exists=True), + type=click.Path(path_type=Path), required=True, help="Directory containing historical KPI files", ) @@ -730,85 +647,37 @@ def analyse_kpis_cmd( status_file: Path | None, ) -> None: """Analyze KPIs for orchestration (fork/exec).""" - import yaml + import time - status_data = {"success": False} - - try: - # Create a minimal postprocess config for analysis - from projects.caliper.orchestration.postprocess_config import ( - CaliperOrchestrationPostprocessConfig, - ) + import yaml - # Create config with analyze enabled and using provided directories - postprocess_config_dict = { - "analyze": { - "enabled": True, - "output": output.name, # Use the provided output filename - "historical_kpis": str(historical_kpis_dir), # Use provided historical dir - } + # Validate paths exist before proceeding + if not current_kpis_file.exists(): + status_data = { + "success": False, + "error": f"Current KPI file not found: {current_kpis_file}", + "completed_at": time.time(), } + click.echo(f"❌ Current KPI file not found: {current_kpis_file}", err=True) - postprocess_config_obj = CaliperOrchestrationPostprocessConfig.model_validate( - postprocess_config_dict - ) - - output_dir = output.parent - - # Run analysis - from projects.caliper.engine.kpi.analyze import analyze_kpis - - try: - result = analyze_kpis( - postprocess_config=postprocess_config_obj, - plugin_module=plugin_module, - base_dir=output_dir, # Use output dir as base since we have absolute paths - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) - except Exception as e: - # Don't suppress the exception - let it bubble up with full traceback - import sys - import traceback - - print(f"❌ Exception in analyze_kpis: {e}", file=sys.stderr) - print(f"Full traceback:\n{traceback.format_exc()}", file=sys.stderr) - raise - - # Check if the analysis function returned a proper result - if not isinstance(result, dict) or "status" not in result: - raise ValueError(f"Invalid result from analyze_kpis: {result}") + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + sys.exit(1) + if not historical_kpis_dir.exists(): status_data = { - "success": result.get("status") == "success", - "output_file": result.get("output_file"), + "success": False, + "error": f"Historical KPI directory not found: {historical_kpis_dir}", + "completed_at": time.time(), } + click.echo(f"❌ Historical KPI directory not found: {historical_kpis_dir}", err=True) - if result.get("status") == "success": - click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") - else: - # Show the full result for debugging if no clear error message - if "error" in result: - error_msg = result["error"] - else: - error_msg = f"Analysis failed with result: {result}" - - status_data["error"] = error_msg - status_data.update(result) # Include all result data for debugging - click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) - if not status_file: - sys.exit(1) - - except Exception as e: # noqa: BLE001 - import traceback - - full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} - click.echo(f"❌ analyse-kpis failed: {e}", err=True) - click.echo(f"Full traceback:\n{full_traceback}", err=True) - if not status_file: - sys.exit(2) - finally: # Write status file if requested if status_file: try: @@ -817,6 +686,85 @@ def analyse_kpis_cmd( except Exception as status_err: click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) sys.exit(4) + sys.exit(2) + + # Delegate to engine layer + from projects.caliper.engine.kpi.analyze import analyze_kpis + + result = analyze_kpis( + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + output_file=output, + plugin_module=plugin_module, + ) + + # Format status data for CLI + status_data = { + "success": result.get("success", False), + "completed_at": result.get("completed_at"), + } + + if result.get("output_file"): + status_data["output_file"] = result["output_file"] + + if result.get("error"): + status_data["error"] = result["error"] + + # Extract detailed analysis results for status file + if result.get("success") and result.get("output_file"): + try: + import yaml + + with open(result["output_file"]) as f: + analysis_report = yaml.safe_load(f) + + # Extract key fields from the analysis report + if analysis_report: + processed = analysis_report.get("processed", {}) + tested = analysis_report.get("tested", {}) + overall = analysis_report.get("overall", {}) + results = analysis_report.get("results", []) + + status_data.update( + { + "baseline_source_count": processed.get("baseline_source_count", 0), + "tested": { + "total_kpis": tested.get("total_kpis", 0), + "regressions": tested.get("regressions", 0), + "passes": tested.get("passes", 0), + "skipped": tested.get("skipped", 0), + }, + "results": results, + "overall": { + "verdict": overall.get("verdict", "UNKNOWN"), + "regression_count": overall.get("regression_count", 0), + "total_tested": overall.get("total_tested", 0), + "total_skipped": overall.get("total_skipped", 0), + }, + } + ) + except Exception as e: + # If we can't read the analysis report, just include basic info + status_data["analysis_report_error"] = f"Could not read analysis report: {e}" + + if result.get("result"): + status_data.update(result["result"]) # Include analysis result for debugging + + # Display result + if result.get("success"): + click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") + else: + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) + + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) # Exit with error code if operation failed if not status_data.get("success", False): diff --git a/projects/caliper/cli/kpi_analysis.py b/projects/caliper/cli/kpi_analysis.py index 7acc5ca84..4eeee8d77 100644 --- a/projects/caliper/cli/kpi_analysis.py +++ b/projects/caliper/cli/kpi_analysis.py @@ -33,21 +33,34 @@ ) def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module: str) -> None: """CLI entrypoint for KPI analysis.""" - from projects.caliper.engine.kpi.analyze import run_kpi_analysis + from projects.caliper.engine.kpi.analyze import ( + analyze_kpis_with_format_conversion, + status_dict_to_exit_code, + ) try: - # Call the core analysis function from engine - exit_code = run_kpi_analysis( - current_kpi_file=current, - historical_data_dir=historical_dir, + # Call the core analysis function from engine (returns status dict) + result = analyze_kpis_with_format_conversion( + current_kpis_file=current, + historical_kpis_dir=historical_dir, output_file=output, plugin_module=plugin_module, ) - if exit_code == 0: + # Convert status dict to exit code for CLI + exit_code = status_dict_to_exit_code(result) + + if result.get("success"): click.echo(f"✅ Analysis completed successfully. Results written to: {output}") + if result.get("regressions_detected"): + click.echo("⚠️ Regressions detected in analysis results") + elif result.get("message"): + click.echo(f"ℹ️ {result.get('message')}") else: - click.echo(f"❌ Analysis failed with exit code {exit_code}", err=True) + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ Analysis failed: {error_msg}", err=True) + + if exit_code != 0: sys.exit(exit_code) except Exception as e: diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 0cccaeca4..f3e16902d 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -14,7 +14,6 @@ analyse_kpis_cmd, artifacts_export, artifacts_import, - kpi_analyze, kpi_csv_export, kpi_generate, kpi_import, @@ -151,7 +150,7 @@ def main( @main.group("kpi") @click.pass_context def kpi_group(ctx: click.Context) -> None: - """KPI generate/import/export/analyze.""" + """KPI generate/import/export/.""" @main.group("artifacts") @@ -211,7 +210,6 @@ def run_cli() -> None: kpi_group.add_command(kpi_generate) kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) -kpi_group.add_command(kpi_analyze) kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 593f8acdb..a6430af31 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -188,7 +188,6 @@ def _build_yaml_report( ) -> dict[str, Any]: """Build the final YAML-serializable report structure.""" regressions = [r for r in results if r.regression] - improvements = [r for r in results if not r.regression and r.relative_change != 0] if regressions: overall_status = "REGRESSION_DETECTED" @@ -250,7 +249,7 @@ def run_kpi_analysis( historical_data_dir: Path, output_file: Path, plugin_module: str, -) -> int: +) -> dict[str, Any]: """Run KPI regression analysis and generate a YAML report. Args: @@ -260,7 +259,7 @@ def run_kpi_analysis( plugin_module: Plugin module name (for loading analysis config) Returns: - Exit code: 0=pass, 1=error, 2=warning (no baseline), 3=regression detected + Status dictionary with success/error information and exit_code for CLI compatibility """ try: logger.info("Running KPI regression analysis") @@ -271,11 +270,23 @@ def run_kpi_analysis( if not current_kpi_file.exists(): logger.error("Current KPI file not found: %s", current_kpi_file) - return 1 + return { + "status": "failed", + "success": False, + "error": f"Current KPI file not found: {current_kpi_file}", + "exit_code": 1, + "completed_at": time.time(), + } if not historical_data_dir.exists(): logger.error("Historical data directory not found: %s", historical_data_dir) - return 1 + return { + "status": "failed", + "success": False, + "error": f"Historical data directory not found: {historical_data_dir}", + "exit_code": 1, + "completed_at": time.time(), + } config = _load_analysis_config(plugin_module) logger.info( @@ -290,18 +301,37 @@ def run_kpi_analysis( if current_data.get("schema_version") != "2": logger.error("Current KPI file must be schema_version 2 (hierarchical)") - return 1 + return { + "status": "failed", + "success": False, + "error": "Current KPI file must be schema_version 2 (hierarchical)", + "exit_code": 1, + "completed_at": time.time(), + } current_records = _extract_kpi_records_from_hierarchical(current_data) if not current_records: logger.warning("No KPI records found in current file") - return 1 + return { + "status": "failed", + "success": False, + "error": "No KPI records found in current file", + "exit_code": 1, + "completed_at": time.time(), + } # Load baseline KPIs baseline_kpi_data = find_baseline_kpis(historical_data_dir) if not baseline_kpi_data: _write_warning_report(output_file, current_kpi_file, plugin_module) - return 2 + return { + "status": "warning", + "success": True, + "message": "no historical KPI found for regression testing", + "output_file": str(output_file), + "exit_code": 2, + "completed_at": time.time(), + } # Build baseline index baseline_index = _build_baseline_index(baseline_kpi_data, config) @@ -367,11 +397,33 @@ def run_kpi_analysis( regressions, ) - return 3 if regressions > 0 else 0 + if regressions > 0: + return { + "status": "regression_detected", + "success": True, + "regressions_detected": True, + "output_file": str(output_file), + "exit_code": 3, + "completed_at": time.time(), + } + else: + return { + "status": "success", + "success": True, + "output_file": str(output_file), + "exit_code": 0, + "completed_at": time.time(), + } - except Exception: + except Exception as e: logger.exception("KPI analysis failed") - return 1 + return { + "status": "failed", + "success": False, + "error": f"KPI analysis failed: {e}", + "exit_code": 1, + "completed_at": time.time(), + } def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: @@ -401,81 +453,6 @@ def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_modu yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) -def analyze_kpis( - postprocess_config, # CaliperOrchestrationPostprocessConfig - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Run KPI analysis step and return result status. - - This is the orchestration interface for KPI analysis. - """ - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled"} - - analyze_config = postprocess_config.analyze - - historical_kpis_dir = Path(analyze_config.historical_kpis) - if not historical_kpis_dir.is_absolute(): - historical_kpis_dir = output_dir / historical_kpis_dir - - output_path = output_dir / analyze_config.output - - try: - if not current_kpis_file.exists(): - return { - "status": "failed", - "error": f"Current KPI file not found: {current_kpis_file}", - "completed_at": time.time(), - } - - if not historical_kpis_dir.exists(): - return { - "status": "failed", - "error": f"Historical KPIs directory not found: {historical_kpis_dir}", - "completed_at": time.time(), - } - - exit_code = run_kpi_analysis( - current_kpi_file=current_kpis_file, - historical_data_dir=historical_kpis_dir, - output_file=output_path, - plugin_module=plugin_module, - ) - - if exit_code == 0: - return { - "status": "success", - "output_file": str(output_path), - "completed_at": time.time(), - } - elif exit_code == 2: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "output_file": str(output_path), - "completed_at": time.time(), - } - elif exit_code == 3: - return { - "status": "regression_detected", - "output_file": str(output_path), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), - } - - except Exception as e: - logger.exception("Analysis step failed") - raise RuntimeError(f"KPI analysis failed: {e}") from e - - def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: """Load all kpis.json files from historical directory. @@ -505,8 +482,21 @@ def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: baseline_kpis[kpi_file] = kpi_data logger.debug("Loaded baseline: %s", kpi_file) - except json.JSONDecodeError as e: - logger.error("Failed to parse JSON in %s: %s", kpi_file, e) + except json.JSONDecodeError: + # Try parsing as JSONL (v1 format) fallback + try: + v1_records = [] + with open(kpi_file) as f: + for line in f: + if line.strip(): + v1_records.append(json.loads(line)) + + if v1_records: + v2_data = _convert_v1_to_v2(v1_records) + baseline_kpis[kpi_file] = v2_data + logger.debug("Loaded baseline (converted from v1): %s", kpi_file) + except Exception: + logger.error("Failed to parse %s as JSON or JSONL", kpi_file) except Exception as e: logger.error("Failed to load %s: %s", kpi_file, e) @@ -534,21 +524,264 @@ def run_analyze( plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - exit_code = run_kpi_analysis( + result = run_kpi_analysis( current_kpi_file=Path(current_path), historical_data_dir=historical_dir, output_file=Path(output_path), plugin_module=plugin_module, ) - if exit_code == 0: - return {"status": "success", "completed_at": time.time()} - elif exit_code == 2: - return {"status": "warning", "message": "no baselines found", "completed_at": time.time()} - elif exit_code == 3: - return {"status": "regression_detected", "completed_at": time.time()} + # Extract just the relevant fields for this legacy interface + return { + "status": result.get("status"), + "message": result.get("message"), + "completed_at": result.get("completed_at"), + } + + +def _convert_v1_to_v2(v1_records: list[dict]) -> dict: + """Convert v1 JSONL records to v2 hierarchical format.""" + metrics = {} + for record in v1_records: + # Extract metric name and value + metric_name = record.get("name", "unknown_metric") + value = record.get("value") + unit = record.get("unit") + labels = record.get("labels", {}) + + # Create metric entry in v2 format + metric_entry = { + "value": value, + "unit": unit, + "labels": labels, + } + + # Determine if higher is better from labels or default to false + higher_is_better = labels.get("higher_is_better", False) + metric_entry["higher_is_better"] = higher_is_better + + metrics[metric_name] = metric_entry + + return { + "schema_version": "2", + "metrics": metrics, + "metadata": { + "converted_from_v1": True, + "original_record_count": len(v1_records), + }, + } + + +def ensure_kpi_file_v2_format(file_path: Path) -> tuple[Path, bool]: + """Convert KPI file to v2 hierarchical format if it's in v1 JSONL format. + + Args: + file_path: Path to KPI file (v1 or v2 format) + + Returns: + Tuple of (converted_file_path, is_temporary_file) + - converted_file_path: Path to v2 format file + - is_temporary_file: True if a temporary file was created and needs cleanup + """ + import json + import tempfile + + logger.debug("Checking format of KPI file: %s", file_path) + + try: + with open(file_path) as f: + content = f.read().strip() + if not content: + raise ValueError(f"KPI file {file_path} is empty") + + # Try to parse as v2 (hierarchical JSON) first + try: + data = json.loads(content) + if isinstance(data, dict) and "schema_version" in data: + # Already in v2 format + logger.debug("KPI file %s is already in v2 format", file_path) + return file_path, False + elif isinstance(data, list): + raise ValueError(f"KPI file {file_path} appears to be in unexpected list format") + else: + # Single dict but missing schema_version - might be v1 single record + logger.debug( + "KPI file %s appears to be single record without schema_version", file_path + ) + except json.JSONDecodeError: + # Not valid single JSON, might be JSONL (v1 format) + logger.debug("KPI file %s not valid single JSON, checking JSONL format", file_path) + + # Try to parse as v1 (JSONL) format + lines = content.split("\n") + non_empty_lines = [line.strip() for line in lines if line.strip()] + + if len(non_empty_lines) == 0: + raise ValueError(f"KPI file {file_path} contains no data") + + # Try to parse first line to validate v1 format + try: + first_record = json.loads(non_empty_lines[0]) + if not isinstance(first_record, dict): + raise ValueError(f"KPI file {file_path} first line is not a JSON object") + + # Parse all lines as v1 JSONL records + v1_records = [] + for i, line in enumerate(non_empty_lines): + try: + record = json.loads(line) + v1_records.append(record) + except json.JSONDecodeError as line_err: + raise ValueError( + f"KPI file {file_path} line {i + 1} has invalid JSON: {line_err}" + ) from line_err + + # Convert v1 records to v2 hierarchical format + logger.info( + "Converting KPI file %s from v1 (JSONL) to v2 (hierarchical) format", file_path + ) + + # Transform to v2 + v2_data = _convert_v1_to_v2(v1_records) + + # Write v2 format to temporary file + temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + try: + json.dump(v2_data, temp_file, indent=2, ensure_ascii=False) + temp_file.flush() + temp_path = Path(temp_file.name) + logger.info( + "Converted %s (%d records) to v2 format at %s", + file_path, + len(v1_records), + temp_path, + ) + return temp_path, True + except Exception as write_err: + temp_file.close() + Path(temp_file.name).unlink(missing_ok=True) + raise ValueError( + f"Failed to write converted v2 format file: {write_err}" + ) from write_err + finally: + temp_file.close() + + except Exception as v1_err: + raise ValueError( + f"KPI file {file_path} is not in valid v1 (JSONL) or v2 (hierarchical) format: {v1_err}" + ) from v1_err + + except Exception as format_err: + raise ValueError( + f"Failed to process KPI file format for {file_path}: {format_err}" + ) from format_err + + +def analyze_kpis( + current_kpis_file: Path, + historical_kpis_dir: Path, + output_file: Path, + plugin_module: str, +) -> dict[str, Any]: + """Analyze KPIs with automatic v1/v2 format conversion. + + This function handles: + 1. Format detection and conversion from v1 (JSONL) to v2 (hierarchical) + 2. Plugin loading + 3. Analysis execution + 4. Temporary file cleanup + 5. Result formatting + + Args: + current_kpis_file: Path to current KPIs file (v1 or v2 format) + historical_kpis_dir: Directory containing historical KPI files + output_file: Path where analysis results will be written + plugin_module: Plugin module name for KPI definitions and analysis rules + + Returns: + Dictionary with status information: + - success: bool - whether analysis completed successfully + - output_file: str - path to analysis results (if successful) + - error: str - error message (if failed) + """ + + try: + # Use directory-based analysis (supports multiple baseline files) + result = run_kpi_analysis( + current_kpi_file=current_kpis_file, + historical_data_dir=historical_kpis_dir, + output_file=output_file, + plugin_module=plugin_module, + ) + + # Convert to the expected format for orchestration and CLI + if result.get("status") == "success": + return { + "status": "success", + "success": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "warning": + return { + "status": "warning", + "success": True, # Warning is still success + "message": result.get("message"), + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "regression_detected": + return { + "status": "regression_detected", + "success": True, # Regression detection is successful analysis + "regressions_detected": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + else: + return { + "status": "failed", + "success": False, + "error": result.get("error", "Analysis failed"), + "completed_at": result.get("completed_at"), + } + + except Exception as e: + logger.exception("KPI analysis with format conversion failed") + return { + "success": False, + "error": str(e), + "completed_at": time.time(), + } + + +def status_dict_to_exit_code(status_dict: dict[str, Any]) -> int: + """Convert a status dictionary to an exit code for CLI use. + + Args: + status_dict: Status dictionary from engine analysis functions + + Returns: + Exit code: 0=success, 1=error, 2=warning (no baseline), 3=regression detected + """ + # If the status dict includes an explicit exit_code, use it + if "exit_code" in status_dict: + return status_dict["exit_code"] + + # Otherwise, determine exit code from status fields + if not status_dict.get("success", False): + return 1 # General error + + status = status_dict.get("status", "") + + if status == "success": + return 0 + elif status == "warning" or "no baseline" in status_dict.get("message", "").lower(): + return 2 # Warning (no baseline data) + elif status == "regression_detected" or status_dict.get("regressions_detected"): + return 3 # Regression detected else: - return {"status": "failed", "error": f"exit code {exit_code}", "completed_at": time.time()} + return 0 # Default to success if unclear # EOF diff --git a/projects/caliper/engine/kpi/analyze_hierarchical.py b/projects/caliper/engine/kpi/analyze_hierarchical.py index c76a9514d..88d56ee92 100644 --- a/projects/caliper/engine/kpi/analyze_hierarchical.py +++ b/projects/caliper/engine/kpi/analyze_hierarchical.py @@ -11,6 +11,59 @@ logger = logging.getLogger(__name__) +def _load_kpis_any_format(kpi_file_path: Path) -> dict[str, Any]: + """Load KPIs from file, handling both v1 (JSONL) and v2 (hierarchical) formats. + + Returns: + Dictionary in hierarchical format with schema_version and metrics + """ + with open(kpi_file_path) as f: + content = f.read().strip() + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Already hierarchical format + return data + else: + # Unknown JSON format, convert list to v1 JSONL handling + raise ValueError("Unknown JSON format") + + except (json.JSONDecodeError, ValueError): + # Try to parse as JSONL (v1 format) + kpis = [] + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + # Convert v1 format to hierarchical format + metrics = {} + for kpi in kpis: + kpi_id = kpi.get("kpi_id") + if kpi_id: + value = kpi.get("value") + labels = kpi.get("labels", {}) + higher_is_better = labels.get("higher_is_better", False) + + metrics[kpi_id] = { + "value": value, + "higher_is_better": higher_is_better, + "labels": labels, + "unit": kpi.get("unit"), + "run_id": kpi.get("run_id"), + "timestamp": kpi.get("timestamp"), + } + + return {"schema_version": "2", "metrics": metrics, "converted_from": "v1_jsonl"} + + def analyze_hierarchical_kpis( current_kpis_path: Path, baseline_kpis_path: Path, @@ -30,31 +83,15 @@ def analyze_hierarchical_kpis( Analysis result dictionary with status, findings, etc. """ try: - # Load KPI files - with open(current_kpis_path) as f: - current_data = json.load(f) - - with open(baseline_kpis_path) as f: - baseline_data = json.load(f) - - # Validate format - if not isinstance(current_data, dict) or not isinstance(baseline_data, dict): - return { - "status": "failed", - "error": "KPI files must be in hierarchical format (dict), not list format", - "completed_at": time.time(), - } - - # Extract schema version and validate - current_schema = current_data.get("schema_version", "unknown") - baseline_schema = baseline_data.get("schema_version", "unknown") - - if current_schema != "2" or baseline_schema != "2": - return { - "status": "failed", - "error": f"Only schema_version 2 supported (current: {current_schema}, baseline: {baseline_schema})", - "completed_at": time.time(), - } + # Load KPI files (supports both v1 JSONL and v2 hierarchical formats) + current_data = _load_kpis_any_format(current_kpis_path) + baseline_data = _load_kpis_any_format(baseline_kpis_path) + + # Both files are now guaranteed to be in hierarchical format with schema_version 2 + logger.info(f"Loaded current KPIs: {current_data.get('converted_from', 'v2_hierarchical')}") + logger.info( + f"Loaded baseline KPIs: {baseline_data.get('converted_from', 'v2_hierarchical')}" + ) # Analyze metrics using plugin-aware logic current_metrics = current_data.get("metrics", {}) @@ -172,8 +209,12 @@ def analyze_hierarchical_kpis( output_path.parent.mkdir(parents=True, exist_ok=True) with open(output_path, "w") as f: json.dump(analysis_result, f, indent=2) + f.write("\n") - logger.info(f"Analysis completed: {regressions} regressions, {improvements} improvements") + metrics_tested = len([m for m in current_metrics if m in baseline_metrics]) + logger.info( + f"Analysis completed: {metrics_tested} verifications, {regressions} regressions, {improvements} improvements" + ) return { "status": "success", @@ -210,5 +251,63 @@ def find_most_recent_baseline(historical_dir: Path) -> Path | None: """Find the most recently modified kpis.json file in historical directory.""" kpi_files = list(historical_dir.rglob("kpis.json")) if not kpi_files: + logger.warning(f"No kpis.json files found in: {historical_dir}") return None - return max(kpi_files, key=lambda p: p.stat().st_mtime) + + logger.info(f"Discovered {len(kpi_files)} baseline candidate files:") + + # Check format of each file and log results + v1_files = [] + v2_files = [] + failed_files = [] + + for kpi_file in kpi_files: + try: + with open(kpi_file) as f: + content = f.read().strip() + + try: + data = json.loads(content) + if isinstance(data, dict) and data.get("schema_version") == "2": + v2_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v2_hierarchical") + else: + # Try parsing as JSONL (v1) + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (unknown format)" + ) + except (json.JSONDecodeError, ValueError): + # Try as JSONL + try: + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (invalid JSONL)" + ) + except Exception: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (parse error)" + ) + except Exception as e: + failed_files.append(kpi_file) + logger.error(f" {kpi_file.relative_to(historical_dir)} → FAILED ({e})") + + logger.info( + f"Summary: {len(v2_files)} v2_hierarchical, {len(v1_files)} v1_jsonl, {len(failed_files)} failed" + ) + + most_recent = max(kpi_files, key=lambda p: p.stat().st_mtime) + logger.info(f"Selected most recent: {most_recent.relative_to(historical_dir)}") + + return most_recent diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index ee7ea1409..009cb0e94 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -291,6 +291,7 @@ def read_kpis_from_file(file_path: Path) -> list[dict]: } run_id = test_metadata.get("run_id") + flat_kpi = { "schema_version": "1", "kpi_id": kpi_record.get("id"), diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index 27dd575c2..99abacada 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -2,10 +2,10 @@ from __future__ import annotations -import json from pathlib import Path from typing import Any +from projects.caliper.engine.kpi.format import write_kpis_in_format from projects.caliper.engine.parse import run_parse from projects.caliper.engine.validation import load_schema, schema_path, validate_instance @@ -18,7 +18,22 @@ def run_kpi_generate( output: Path | None, use_cache: bool, cache_path: Path | None, + format_type: str = "hierarchical", ) -> list[dict[str, Any]]: + """Generate KPI output in specified format. + + Args: + base_dir: Directory containing test artifacts + plugin_module: Name of the plugin module + plugin: Plugin instance + output: Path to write output file + use_cache: Whether to use cached parse results + cache_path: Path to cache file (not used currently) + format_type: Output format - "hierarchical" (default) or "jsonl" + + Returns: + List of KPI records + """ model = run_parse( base_dir=base_dir, plugin_module=plugin_module, @@ -30,7 +45,8 @@ def run_kpi_generate( kpi_schema = load_schema(schema_path("kpi_record.schema.json")) for row in rows: validate_instance(row, kpi_schema, "KPI record") - text = "\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + ("\n" if rows else "") + if output: - output.write_text(text, encoding="utf-8") + write_kpis_in_format(rows, output, format_type, model) + return rows diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 70430ebd5..a46eb906a 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -24,6 +24,7 @@ ) from projects.caliper.orchestration.cli_builder import ( build_ai_eval_export_command, + build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, build_parse_command, @@ -120,117 +121,6 @@ def _resolve_visualize_config_path( # _transform_kpis_to_hierarchical_format function removed - CLI commands now handle format conversion - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - test_data = tests_data[run_id] - - # Extract common labels (excluding KPI-specific ones) - kpi_labels = kpi.get("labels", {}) - test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels - - # Merge test labels (they should be the same for all KPIs in a test) - if not test_data["labels"]: - test_data["labels"] = test_labels - - # Store test metadata from first KPI - if not test_data["metadata"]: - test_data["metadata"] = { - "timestamp": kpi.get("timestamp"), - "source": kpi.get("source", {}), - "run_id": run_id, - } - - # Create KPI record with metadata - kpi_id = kpi.get("kpi_id") - raw_value = kpi.get("value") - - # Transform 2D KPI values to structured format - if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: - # Check if this looks like 2D data: list of tuples/lists with 2 elements - first_item = raw_value[0] - if isinstance(first_item, list | tuple) and len(first_item) == 2: - try: - # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format - structured_value = { - "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], - "count": len(raw_value), - } - final_value = structured_value - except (ValueError, TypeError, IndexError): - # If conversion fails, keep original value - final_value = raw_value - else: - final_value = raw_value - else: - final_value = raw_value - - kpi_record = { - "id": kpi_id, - "value": final_value, - "unit": kpi.get("unit"), - "higher_is_better": kpi_labels.get("higher_is_better", True), - } - - # Add KPI metadata from function decorator if available - if kpi_id in kpi_functions: - func = kpi_functions[kpi_id] - kpi_record.update( - { - "name": ( - func.__doc__.replace(" KPI.", "") - if func.__doc__ - else kpi_id.replace("_", " ").title() - ), - "help": getattr(func, "_kpi_help", ""), - } - ) - - # Add 2D-specific metadata if present - if getattr(func, "_kpi_is_2d", False): - kpi_record.update( - { - "is_2d": True, - "x_unit": getattr(func, "_kpi_x_unit", ""), - "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], - "y_help": getattr(func, "_kpi_y_help", None) - or getattr(func, "_kpi_help", ""), - } - ) - else: - kpi_record["is_2d"] = False - - # Add formatting info if available - if hasattr(func, "_kpi_format"): - kpi_record["format"] = func._kpi_format - else: - # Fallback if no function metadata available - kpi_record.update( - { - "name": kpi_id.replace("_", " ").title(), - "help": f"KPI: {kpi_id}", - "is_2d": isinstance(kpi.get("value"), list), - } - ) - - test_data["kpis"].append(kpi_record) - - # Convert to final structure - tests_list = [] - for run_id, test_data in tests_data.items(): - tests_list.append( - { - "run_id": run_id, - "labels": test_data["labels"], - "metadata": test_data["metadata"], - "kpis": test_data["kpis"], - } - ) - - return {"schema_version": "2", "tests": tests_list} - def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, From 86b6523bea8d1ed7a165f8b5d4d0c2245408c98b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 22:20:58 +0200 Subject: [PATCH 0338/1072] [caliper] tests: test_kpi_analyze: update to use the test status dict --- projects/caliper/tests/test_kpi_analyze.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py index a71e01c3d..2ca06e4de 100644 --- a/projects/caliper/tests/test_kpi_analyze.py +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -197,14 +197,14 @@ def test_pass_no_regression(self, tmp_path): self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline1) self._write_hierarchical_kpi(historical_dir / "run2" / "kpis.json", baseline2) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 0 + assert test_status["exit_code"] == 0 assert output_file.exists() with open(output_file) as f: @@ -247,14 +247,14 @@ def test_regression_detected(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 3 + assert test_status["exit_code"] == 3 with open(output_file) as f: report = yaml.safe_load(f) @@ -282,14 +282,14 @@ def test_no_historical_data(self, tmp_path): ) self._write_hierarchical_kpi(current_dir / "kpis.json", current) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 2 + assert test_status["exit_code"] == 2 with open(output_file) as f: report = yaml.safe_load(f) assert report["overall"]["verdict"] == "NO_BASELINE" @@ -327,14 +327,14 @@ def test_mixed_regression_and_pass(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 3 + assert test_status["exit_code"] == 3 with open(output_file) as f: report = yaml.safe_load(f) @@ -376,14 +376,14 @@ def test_skips_non_scalar_kpis(self, tmp_path): ) self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, plugin_module="nonexistent_plugin", ) - assert exit_code == 0 + assert test_status["exit_code"] == 0 with open(output_file) as f: report = yaml.safe_load(f) @@ -426,7 +426,7 @@ def test_comparison_keys_separate_baselines(self, tmp_path): # With version as comparison_key, both match on platform=A100 # (plugin would expose this config; here we test the core logic directly) - exit_code = run_kpi_analysis( + test_status = run_kpi_analysis( current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, @@ -435,7 +435,7 @@ def test_comparison_keys_separate_baselines(self, tmp_path): # Without plugin config, default AnalysisConfig has no comparison_keys, # so version must also match → no baselines found → skip - assert exit_code == 0 or exit_code == 2 + assert test_status["exit_code"] in (0, 2) with open(output_file) as f: report = yaml.safe_load(f) # The version label differs, so with empty comparison_keys they don't match From 376e78c83f77453b340d4c71e093b901ed9e7ad3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 31 Jul 2026 22:35:28 +0200 Subject: [PATCH 0339/1072] [skeleton] orchestration: config: fix the analyze config --- projects/skeleton/orchestration/config.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 9ee43c738..d0662ae62 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -77,11 +77,6 @@ caliper: enabled: true output_dir: ai_data - analyze: - enabled: true - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke @@ -104,6 +99,13 @@ caliper: include_kpis_json: true include_ai_data: true + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json + + agentic: enabled: false model_key: qwen-3-6-35b From 823ca895395e7e55c0f5689f66a18b079bf4d46f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 1 Aug 2026 07:54:18 +0200 Subject: [PATCH 0340/1072] [caliper] orchestration: cli_builder: make the command easy to copy-and-paste --- projects/caliper/orchestration/cli_builder.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 7e12fd4f4..51cc03498 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -516,9 +516,7 @@ def log_caliper_start_banner(command: list[str], script_path: Path, step_name: s # Log banner indicating Caliper is starting logger.info("=" * 60) logger.info("🚀 STARTING CALIPER %s (fork/exec)", step_name.upper()) - logger.info("📄 Command:") - for line in formatted_command.splitlines(): - logger.info(" %s", line) + logger.info(f"📄 Command:\n{formatted_command}") logger.info("=" * 60) @@ -563,7 +561,7 @@ def _format_command_for_display(command: list[str]) -> str: lines.append(command[i]) i += 1 - return "\n".join(lines) + return f" \\\n{len('caliper ') * ' '}".join(lines).strip() def handle_caliper_output_and_completion( From de3a282e8224b45dc3b95ee7de9d74dda81d4cac Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Sat, 1 Aug 2026 08:00:40 +0200 Subject: [PATCH 0341/1072] [kserve] toolbox: capture_llmisvc_state: capture the nodes details --- .../toolbox/capture_llmisvc_state/main.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index c921d45be..2daafbddc 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -312,5 +312,126 @@ def capture_pods_describe(args, context): return f"Pod describe output captured for {len(pod_names)} pods" +@task +def determine_used_nodes(args, context): + """Determine which nodes are used by LLMInferenceService pods""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].spec.nodeName}}"', + check=False, + log_stdout=False, + ) + + node_names = list(set(result.stdout.strip().split())) # Remove duplicates + if not node_names or not result.stdout.strip(): + context.used_nodes = [] + return "No nodes found for LLMInferenceService pods" + + context.used_nodes = [node for node in node_names if node] # Filter out empty strings + return f"Found {len(context.used_nodes)} nodes used by LLMInferenceService pods: {', '.join(context.used_nodes)}" + + +@task +def capture_pod_node_mapping(args, context): + """Capture pod->node mapping as YAML""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o custom-columns=POD:.metadata.name,NODE:.spec.nodeName --no-headers', + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return "No pod->node mapping found" + + mapping_file = args.artifact_dir / "artifacts/pod_node_mapping.yaml" + pod_node_mapping = {} + + for line in result.stdout.strip().split("\n"): + if line.strip(): + parts = line.split() + if len(parts) >= 2: + pod_name = parts[0] + node_name = parts[1] if parts[1] != "" else None + pod_node_mapping[pod_name] = node_name + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"pod_node_mapping": pod_node_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Pod->node mapping captured to {mapping_file} ({len(pod_node_mapping)} pods)" + + +@task +def capture_node_gpu_mapping(args, context): + """Capture node->GPU type mapping as YAML""" + if not context.used_nodes: + return "No nodes to capture GPU mapping for" + + node_gpu_mapping = {} + + for node_name in context.used_nodes: + # Try nvidia.com/gpu.product first + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.nvidia\\.com/gpu\\.product}}"', + check=False, + log_stdout=False, + ) + gpu_type = result.stdout.strip() + + # If not found, try gpu.nvidia.com/class as fallback + if not gpu_type: + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.gpu\\.nvidia\\.com/class}}"', + check=False, + log_stdout=False, + ) + gpu_class = result.stdout.strip() + if gpu_class: + gpu_type = f"NVIDIA-{gpu_class}" + else: + gpu_type = "unknown" + + node_gpu_mapping[node_name] = gpu_type + + mapping_file = args.artifact_dir / "artifacts/node_gpu_mapping.yaml" + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"node_gpu_mapping": node_gpu_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Node->GPU mapping captured to {mapping_file} ({len(node_gpu_mapping)} nodes)" + + +@task +def capture_used_nodes_yaml(args, context): + """Capture YAML definitions for nodes used by LLMInferenceService pods""" + if not context.used_nodes: + return "No nodes to capture YAML for" + + nodes_dir = args.artifact_dir / "artifacts/nodes" + shell.mkdir("artifacts/nodes") + + captured_count = 0 + for node_name in context.used_nodes: + node_file = nodes_dir / f"{node_name}.yaml" + shell.run( + f"oc get node {node_name} -oyaml", + stdout_dest=node_file, + check=False, + ) + captured_count += 1 + + return f"Captured YAML for {captured_count} nodes in {nodes_dir}" + + if __name__ == "__main__": run.main() From a9b04840ceffe9555a5f56d8c59acf98dc619333 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:24:54 +0200 Subject: [PATCH 0342/1072] [caliper] orchestration: notification: include the post-processing message status --- .../caliper/orchestration/notification.py | 34 +++++++------------ 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 8901afdf6..9a8158842 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -89,31 +89,21 @@ def format_postprocess_status_notification( else: step_name_display = f"**{step_name}**" - lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") - - # Add step message if available (but not for warning/failed steps to avoid duplication) - if step_result.message and step_result.status not in ("warning", "failed", "failure"): - lines.append(f" > {step_result.message}") - - # Add reason for skipped steps - if step_result.status in ("skipped", "disabled") and step_result.reason: - lines.append(f" > {step_result.reason}") - - # Add error message for failed steps - if ( - step_result.status in ("failed", "failure") - and hasattr(step_result, "error") - and step_result.error - ): - lines.append(f" > ❌ {step_result.error}") + # Format step with message if available + if step_result.message: + lines.append( + f"- {step_emoji} {step_name_display}: `{step_result.status}` * {step_result.message}" + ) + else: + lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") - # Add warning message for warning steps + # Add reason for skipped steps (only if no message already shown) if ( - step_result.status == "warning" - and hasattr(step_result, "message") - and step_result.message + step_result.status in ("skipped", "disabled") + and step_result.reason + and not step_result.message ): - lines.append(f" > ⚠️ {step_result.message}") + lines.append(f" > {step_result.reason}") # Add specific file links for certain steps if step_result.status == "success" and get_file_link: From dd3a6e33818d75d4e115ee444bdede0a10b976c8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:25:31 +0200 Subject: [PATCH 0343/1072] [caliper] orchestration: postprocess: include a message in the post-processing status --- projects/caliper/orchestration/postprocess.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index a46eb906a..9fd7b1927 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -210,7 +210,12 @@ def _run_artifacts_to_kpis( full_traceback = traceback.format_exc() logger.error(f"KPI generation failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } # _run_s3_import function removed - now using fork/exec subprocess execution directly @@ -282,7 +287,12 @@ def _run_artifacts_to_ai_data( full_traceback = traceback.format_exc() logger.error(f"AI eval export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } def _load_test_labels(test_dir: Path) -> dict[str, Any]: @@ -396,7 +406,12 @@ def _run_kpis_to_csv( full_traceback = traceback.format_exc() logger.error(f"KPI CSV export failed: {e}") logger.error(f"Full traceback:\n{full_traceback}") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } def _run_analyse_kpis( @@ -477,7 +492,12 @@ def _run_analyse_kpis( except Exception as e: logger.exception("KPI analysis failed in _run_analyse_kpis") - return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } class CaliperPostprocessOrchestrator: @@ -672,7 +692,7 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A warning_msg = result.get("message", "unknown warning") logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") elif status == "failed": - error_msg = result.get("error", "unknown error") + error_msg = result.get("message", "unknown error") traceback_msg = result.get("traceback") logger.error(f"Step '{step_name}' failed: {error_msg}") if traceback_msg: From 2769596bc3b48528333187842d7631fe79d6c04f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:38:23 +0200 Subject: [PATCH 0344/1072] fournos: gitops: base/workflows/pipeline-replot: use 0x instead of 00x --- fournos/gitops/base/workflows/pipeline-replot.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fournos/gitops/base/workflows/pipeline-replot.yaml b/fournos/gitops/base/workflows/pipeline-replot.yaml index 952407f02..07fdc5b92 100644 --- a/fournos/gitops/base/workflows/pipeline-replot.yaml +++ b/fournos/gitops/base/workflows/pipeline-replot.yaml @@ -17,7 +17,7 @@ spec: - name: kubeconfig-secret type: string tasks: - - name: 000-replot + - name: 00-replot taskRef: name: forge-step kind: Task @@ -34,10 +34,10 @@ spec: - name: job-step value: replot - name: job-step-name - value: 000__replot + value: 00__replot finally: - - name: 001-export-artifacts + - name: 01-export-artifacts taskRef: name: forge-step kind: Task @@ -54,4 +54,4 @@ spec: - name: job-step value: export-artifacts - name: job-step-name - value: 001__export-artifacts + value: 01__export-artifacts From 9cc931b55149f0eeadde4766e400873f213fa594 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:49:23 +0200 Subject: [PATCH 0345/1072] [caliper] cli: commands: fail early if no data is parsed --- projects/caliper/cli/commands.py | 50 ++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 22be577ec..849d2f0d7 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -478,27 +478,59 @@ def kpi_generate( status_data = {"success": False} try: - run_kpi_generate( + # First run parse to check test directory count + model = run_parse( base_dir=artifact_root, plugin_module=mod, plugin=plugin, - output=output, use_cache=True, - cache_path=None, - format_type=format_type, ) - status_data = {"success": True, "output_file": str(output)} - click.echo(f"Generated {output}") + + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + test_dir_count = len(test_directories) + + # Check if any test directories were found + if test_dir_count == 0: + status_data = { + "success": False, + "message": "No test directories found - nothing to process for KPI generation", + "test_directories_count": 0, + "test_directories": [], + } + click.echo("❌ No test directories found - KPI generation failed", err=True) + click.echo(" No __test_labels__.yaml files found in artifact directory", err=True) + + sys.exit(3) + else: + # Proceed with KPI generation + run_kpi_generate( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, + format_type=format_type, + ) + status_data = { + "success": True, + "output_file": str(output), + "test_directories_count": test_dir_count, + "test_directories": test_directories, + } + click.echo(f"Generated {output}") + click.echo(f"📁 Processed {test_dir_count} test directories") + except Exception as e: # noqa: BLE001 import traceback full_traceback = traceback.format_exc() - status_data = {"success": False, "error": str(e), "traceback": full_traceback} + status_data = {"success": False, "message": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) click.echo(f"Full traceback:\n{full_traceback}", err=True) - if not status_file: - sys.exit(3) + sys.exit(3) finally: # Write status file if requested if status_file: From d4c1e14613aef41b6321b81f497a64477a8ca4a2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 09:49:44 +0200 Subject: [PATCH 0346/1072] [caliper] orchestration: replot: raise the warning level for visibility --- projects/caliper/orchestration/replot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/replot.py b/projects/caliper/orchestration/replot.py index 86b6382a4..fd2132327 100644 --- a/projects/caliper/orchestration/replot.py +++ b/projects/caliper/orchestration/replot.py @@ -187,7 +187,7 @@ def run_replot_from_orchestration_config( if skip_download: keep_replot_dir = True - logger.info("No replot URL provided, skipping artifact download") + logger.warning("No replot URL provided, skipping artifact download") logger.info("Automatically setting keep_replot_dir=True to preserve existing files") logger.info(f"Download directory: {replot_download_dir}") logger.info(f"Output directory: {artifact_directory}") From b70548b421edfa6fc7d21967ca75c9c54eae0e32 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Tue, 21 Jul 2026 16:54:41 +0100 Subject: [PATCH 0347/1072] feat: Fix timeout flow Signed-off-by: Alberto Perdomo --- .../tests/test_run_guidellm_benchmark.py | 8 ++++ .../toolbox/run_guidellm_benchmark/main.py | 38 ++++++++++++++++++- .../toolbox/run_guidellm_benchmark/utils.py | 3 ++ .../orchestration/config.d/workloads.yaml | 3 +- projects/llm_d/tests/test_profiles.py | 5 +++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/projects/guidellm/tests/test_run_guidellm_benchmark.py b/projects/guidellm/tests/test_run_guidellm_benchmark.py index b867e72bf..29d4c6b92 100644 --- a/projects/guidellm/tests/test_run_guidellm_benchmark.py +++ b/projects/guidellm/tests/test_run_guidellm_benchmark.py @@ -3,6 +3,7 @@ import pytest from projects.guidellm.toolbox.run_guidellm_benchmark import build_guidellm_args +from projects.guidellm.toolbox.run_guidellm_benchmark.main import _wait_attempts from projects.guidellm.toolbox.run_guidellm_benchmark.utils import ( expand_guidellm_runs, render_guidellm_job_from_parts, @@ -114,6 +115,7 @@ def test_render_guidellm_job_from_parts_uses_shell_for_multi_run_benchmarks() -> name="guidellm-benchmark", image="ghcr.io/vllm-project/guidellm:v0.5.4", endpoint_url="https://example.test/llm-d", + timeout_seconds=3600, guidellm_args=[ "--backend-type=openai_http", "--rate-type=concurrent", @@ -124,6 +126,7 @@ def test_render_guidellm_job_from_parts_uses_shell_for_multi_run_benchmarks() -> ) container = manifest["spec"]["template"]["spec"]["containers"][0] + assert manifest["spec"]["activeDeadlineSeconds"] == 3600 assert container["command"] == ["/bin/sh", "-lc"] script = container["args"][0] assert "--rate=32" in script @@ -142,6 +145,7 @@ def test_render_guidellm_job_from_parts_keeps_plain_rates_as_single_guidellm_run name="guidellm-benchmark", image="ghcr.io/vllm-project/guidellm:v0.5.4", endpoint_url="https://example.test/llm-d", + timeout_seconds=3600, guidellm_args=[ "--backend-type=openai_http", "--rate-type=concurrent", @@ -165,6 +169,10 @@ def test_render_guidellm_job_from_parts_keeps_plain_rates_as_single_guidellm_run ] +def test_wait_attempts_cover_the_job_timeout_and_status_grace_period() -> None: + assert _wait_attempts(3600) == 367 + + def test_build_guidellm_args_renders_list_values() -> None: benchmark = { "outputs": "json", diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 65cafe544..ea4ea0cdc 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -5,7 +5,9 @@ import gzip import json import logging +import math import subprocess +import time from pathlib import Path from projects.core.dsl import entrypoint, execute_tasks, retry, task @@ -23,6 +25,8 @@ ) logger = logging.getLogger(__name__) +WAIT_POLL_INTERVAL_SECONDS = 10 +JOB_COMPLETION_GRACE_SECONDS = 60 def trim_benchmark_json(obj): @@ -64,7 +68,7 @@ def run( name: Name of the benchmark job namespace: Namespace to run the benchmark job in (empty string auto-detects current namespace) image: Full container image reference for the benchmark - timeout: Timeout in seconds to wait for job completion + timeout: Active deadline for the Job and timeout in seconds to wait for completion pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. @@ -72,10 +76,23 @@ def run( keep_full_benchmark_file: Whether to keep the full untrimmed benchmark JSON files alongside trimmed ones (default: False) """ + if timeout <= 0: + raise ValueError("timeout must be greater than zero") + + # Allow time for Kubernetes to record the Job's terminal status after its deadline. + wait_guidellm_benchmark_task._retry_config["attempts"] = _wait_attempts(timeout) execute_tasks(locals()) return 0 +def _wait_attempts(timeout_seconds: int) -> int: + return max( + 1, + math.ceil((timeout_seconds + JOB_COMPLETION_GRACE_SECONDS) / WAIT_POLL_INTERVAL_SECONDS) + + 1, + ) + + @task def validate_parameters(args, ctx): """Validate and normalize parameters""" @@ -157,6 +174,7 @@ def create_guidellm_resources_task(args, ctx): image=ctx.image, endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, + timeout_seconds=args.timeout, hf_token_secret=args.hf_token_secret, fs_group=args.fs_group, ), @@ -186,10 +204,11 @@ def create_guidellm_resources_task(args, ctx): ), ) + ctx.wait_deadline = time.monotonic() + args.timeout + JOB_COMPLETION_GRACE_SECONDS return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" -@retry(attempts=360, delay=30, backoff=1.0) +@retry(attempts=366, delay=WAIT_POLL_INTERVAL_SECONDS, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" @@ -209,6 +228,8 @@ def wait_guidellm_benchmark_task(args, ctx): active = active_result.stdout.strip() == "1" if active_result.returncode == 0 else False if active: + if time.monotonic() >= ctx.wait_deadline: + _raise_benchmark_timeout(args, ctx) logger.info("Job %s is still active, retrying...", ctx.benchmark_name) return False # Retry immediately @@ -268,9 +289,22 @@ def wait_guidellm_benchmark_task(args, ctx): ) raise RuntimeError(f"GuideLLM job {ctx.benchmark_name} failed") + if time.monotonic() >= ctx.wait_deadline: + _raise_benchmark_timeout(args, ctx) return False # Retry +def _raise_benchmark_timeout(args, ctx) -> None: + capture_guidellm_state( + artifact_dir=args.artifact_dir, + namespace=ctx.target_namespace, + benchmark_name=ctx.benchmark_name, + ) + raise TimeoutError( + f"GuideLLM benchmark {ctx.benchmark_name} did not complete within {args.timeout}s" + ) + + @task def capture_guidellm_state_task(args, ctx): """Capture GuideLLM benchmark job state and logs""" diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 8454f81aa..f63ab8280 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -184,6 +184,7 @@ def render_guidellm_job_from_parts( image: str, endpoint_url: str, guidellm_args: list[str], + timeout_seconds: int, hf_token_secret: str = "", fs_group: int | None = None, ) -> dict[str, Any]: @@ -195,6 +196,7 @@ def render_guidellm_job_from_parts( image: Container image for GuideLLM endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM + timeout_seconds: Active deadline for the Kubernetes Job hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. fs_group: If set, adds a pod-level securityContext.fsGroup to ensure the PVC is writable by the container. Needed on clusters where the @@ -215,6 +217,7 @@ def render_guidellm_job_from_parts( }, ) manifest = yaml.safe_load(rendered_yaml) + manifest["spec"]["activeDeadlineSeconds"] = timeout_seconds container = manifest["spec"]["template"]["spec"]["containers"][0] if len(runs) == 1 and runs[0].rate is None: container["command"] = ["/opt/app-root/bin/guidellm"] diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 0e29ea5eb..89a5bffb9 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -16,7 +16,7 @@ benchmarks: max_model_len: 4096 short: - timeout_seconds: 900 + timeout_seconds: 3600 outputs: json args: backend_type: openai_http @@ -27,6 +27,7 @@ benchmarks: data: prompt_tokens=256,output_tokens=128 concurrent-1k-1k: + timeout_seconds: 3600 args: backend_type: openai_http rate_type: concurrent diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 1efce2f07..1f056dbd1 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -87,12 +87,16 @@ def test_smoke_presets_inherit_deployment_modes(preset: str, expected_deployment def test_benchmark_workloads_are_available() -> None: _init_project_config() + short = core_config.project.get_config("workloads.benchmarks.short", print=False) concurrent = core_config.project.get_config( "workloads.benchmarks.concurrent-1k-1k", print=False ) heavy = core_config.project.get_config("workloads.benchmarks.heavy-heterogeneous", print=False) multi_turn = core_config.project.get_config("workloads.benchmarks.multi-turn", print=False) + for benchmark in (short, concurrent, heavy, multi_turn): + assert benchmark["timeout_seconds"] == 3600 + assert concurrent["args"]["rate"] == [300, 200, 100, 50, 1] assert heavy["args"]["max_seconds"] == 600 assert "prompt_tokens_stdev=8500" in heavy["args"]["data"] @@ -140,6 +144,7 @@ def _fake_run(**kwargs): monkeypatch.setattr(test_phase.run_guidellm_benchmark_command, "run", _fake_run) test_phase.run_guidellm_benchmark(endpoint_url="https://example.test/llm-d") + assert captured["timeout"] == 3600 guidellm_args = captured["guidellm_args"] assert isinstance(guidellm_args, list) assert "--processor=openai/gpt-oss-120b" in guidellm_args From 7d63317cb600ab740516b974afc4108bd57de27c Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Fri, 24 Jul 2026 10:02:17 +0100 Subject: [PATCH 0348/1072] feat: add rhoai release testing matrix --- .../orchestration/config.d/deployments.yaml | 41 +++++++++++++++++-- .../orchestration/config.d/model_cache.yaml | 2 + .../llm_d/orchestration/config.d/runtime.yaml | 1 + .../scheduler_config/__router_template__.yaml | 4 ++ .../precise-prefix-cache.yaml | 9 ++++ .../orchestration/presets.d/presets.yaml | 18 ++++++++ .../orchestration/render_inference_service.py | 27 ++++++++++++ .../llm_d/orchestration/runtime_config.py | 22 ++++++++-- projects/llm_d/tests/test_profiles.py | 39 +++++++++++++++--- 9 files changed, 150 insertions(+), 13 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index a2b29c769..70e26ede2 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -6,10 +6,11 @@ defaults: tensor_parallelism: 1 vllm_extra: args: - - --gpu-memory-utilization=0.92 - - --trust-remote-code - - --no-enable-log-requests - - --enable-prefix-caching + max_model_len: 4096 + gpu_memory_utilization: 0.92 + trust_remote_code: true + no_enable_log_requests: true + enable_prefix_caching: true pd: vllm_extra: @@ -28,12 +29,44 @@ pd: profiles: approximate-prefix-cache: scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml + replicas: 4 + tensor_parallelism: 2 + vllm_extra: + args: + max_model_len: 40960 precise-prefix-cache: scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml + replicas: 4 + tensor_parallelism: 2 + annotations: + benchflow.io/prefix-caching: precise + vllm_extra: + args: + max_model_len: 40960 + prefix_caching_hash_algo: sha256_cbor + block_size: 64 + # The serving image expands VLLM_ADDITIONAL_ARGS through `eval`; literal + # single quotes keep the JSON intact for vLLM after that shell parse. + kv_events_config: "'{\"enable_kv_cache_events\": true, \"publisher\": \"zmq\", \"endpoint\": \"tcp://__INFERENCE_SERVICE_NAME__-epp-service:5557\", \"topic\": \"kv@$(POD_IP)@$(MODEL_NAME)\"}'" + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: MODEL_NAME + value: __MODEL_NAME__ + - name: PYTHONHASHSEED + value: "42" distributed-default: scheduler: {} + replicas: 4 + tensor_parallelism: 2 + vllm_extra: + args: + max_model_len: 40960 # Template flavor configurations with name-based extraction templates: diff --git a/projects/llm_d/orchestration/config.d/model_cache.yaml b/projects/llm_d/orchestration/config.d/model_cache.yaml index eae017723..0c905f127 100644 --- a/projects/llm_d/orchestration/config.d/model_cache.yaml +++ b/projects/llm_d/orchestration/config.d/model_cache.yaml @@ -5,6 +5,8 @@ pvc: name_prefix: llm-d-model size: 15Gi access_mode: ReadWriteOnce + # GitHub /var overrides can select an RWX-capable class together with + # access_mode: ReadWriteMany for multi-node model-cache mounts. storage_class_name: null model_directory_name: model diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index bea48e83a..d092edc41 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -3,6 +3,7 @@ model_name: Qwen/Qwen3-0.6B deployment_profile: approximate-prefix-cache smoke_request_key: default benchmark_key: null +isolate_run_specs: false job_name: null namespace: forge-llm-d kserve: diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml index 8c33cbd30..bdba5de23 100644 --- a/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml +++ b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml @@ -2,6 +2,10 @@ scheduler: template: containers: - name: main + ports: + - name: zmq + containerPort: 5557 + protocol: TCP env: - name: TOKENIZER_CACHE_DIR value: /tmp/tokenizer-cache diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml index 20e8895bf..189b05b3e 100644 --- a/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml +++ b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml @@ -8,13 +8,22 @@ plugins: - type: single-profile-handler - type: precise-prefix-cache-scorer parameters: + kvEventsConfig: + zmqEndpoint: tcp://*:5557 + topicFilter: kv indexerConfig: tokenProcessorConfig: blockSize: 64 hashSeed: "42" tokenizersPoolConfig: + modelName: base hf: tokenizersCacheDir: /tmp/tokenizers + uds: + socketFile: /tmp/tokenizer/tokenizer-uds.socket + modelTokenizerMap: + # KServe mounts the PVC URI's model subpath at this directory. + base: /mnt/models/base - type: kv-cache-utilization-scorer - type: queue-scorer - type: max-score-picker diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 6b42bb701..ce241513d 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -45,3 +45,21 @@ gpt-oss-120b-inference-scheduling-release: - concurrent-1k-1k - heavy-heterogeneous - multi-turn + +llama-33-70b-rhoai-release: + runtime.selected_preset: llama-33-70b-rhoai-release + runtime.isolate_run_specs: true + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic + runtime.deployment_profile: + - distributed-default + - precise-prefix-cache + - approximate-prefix-cache + runtime.smoke_request_key: default + runtime.benchmark_key: + - concurrent-1k-1k + - heavy-heterogeneous + - multi-turn + model_cache.pvc.size: 300Gi + platform.operators.rhods-operator.channel: beta + workloads.args: + request_type: text_completions diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index b27456e9c..819452e32 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -19,6 +19,18 @@ def _load_yaml(path: Path) -> Any: return yaml.safe_load(handle) +def _replace_placeholders(value: Any, replacements: dict[str, str]) -> Any: + """Replace Python-time deployment placeholders while preserving value types.""" + if isinstance(value, dict): + return {key: _replace_placeholders(item, replacements) for key, item in value.items()} + if isinstance(value, list): + return [_replace_placeholders(item, replacements) for item in value] + if isinstance(value, str): + for placeholder, replacement in replacements.items(): + value = value.replace(placeholder, replacement) + return value + + def _create_model_cache_spec( model_cache: dict[str, Any], source_uri: str, @@ -114,6 +126,18 @@ def render_inference_service_from_parts( namespace=namespace, ) + # These sentinels are resolved here because VLLM_ADDITIONAL_ARGS is opaque + # shell input and cannot use controller-time Go-template substitutions. + rendered_service_name = f"llm-d-{deployment_profile_name}" if deployment_profile_name else name + deployment_profile = _replace_placeholders( + deployment_profile, + { + "__INFERENCE_SERVICE_NAME__": rendered_service_name, + "__MODEL_NAME__": model_name.removeprefix("hf://"), + }, + ) + manifest["metadata"]["annotations"].update(deployment_profile.get("annotations", {})) + manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug @@ -269,6 +293,9 @@ def _render_standard_deployment( serving_container["env"] = [] serving_container["env"].append({"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}) + serving_container["env"].extend( + copy.deepcopy(deployment_profile.get("vllm_extra", {}).get("env", [])) + ) # Configure router/scheduler has_scheduler_key = "scheduler" in deployment_profile diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index dcfc9a7f1..f1eb6ba4d 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -12,7 +12,7 @@ import yaml -from projects.core.dsl.utils import slugify_identifier +from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -611,13 +611,25 @@ def get_run_specs() -> list[RunSpec]: benchmark_entries = [(None, None)] combinations = list(product(model_names, profile_names, benchmark_entries)) - namespace = get_namespace() + base_namespace = get_namespace() + isolate_run_specs = bool(_get_runtime_value("isolate_run_specs")) run_specs: list[RunSpec] = [] for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) + if not isolate_run_specs or len(combinations) == 1: + namespace = base_namespace + artifact_dirname = f"llmd__{bench_key or 'default'}" + else: + namespace_parts = [base_namespace, model_slug, profile_slug] + artifact_parts = ["llmd", model_slug, profile_slug] + if bench_slug: + namespace_parts.append(bench_slug) + artifact_parts.append(bench_slug) + namespace = truncate_k8s_name("-".join(namespace_parts), max_length=63) + artifact_dirname = "__".join(artifact_parts) run_specs.append( RunSpec( @@ -628,7 +640,7 @@ def get_run_specs() -> list[RunSpec]: benchmark_key=bench_key, benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=f"llmd__{bench_key or 'default'}__{profile_slug}", + artifact_dirname=artifact_dirname, ) ) @@ -642,18 +654,20 @@ def activate_run_spec(run_spec: RunSpec): saved = { "model_name": _get_runtime_value("model_name"), "deployment_profile": _get_runtime_value("deployment_profile"), - "namespace_override": _get_runtime_value("namespace_override"), + "namespace": _get_runtime_value("namespace"), "benchmark_key": _get_runtime_value("benchmark_key"), } config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) + config.project.set_config("runtime.namespace", run_spec.namespace) config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) try: yield finally: config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) + config.project.set_config("runtime.namespace", saved["namespace"]) config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 1f056dbd1..4a17bc923 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -56,8 +56,8 @@ def test_release_deployment_profiles_have_expected_shape() -> None: distributed = runtime_config.get_deployment_profile() for profile in (approximate, precise, distributed): - assert profile["replicas"] == 1 - assert profile["tensor_parallelism"] == 1 + assert profile["replicas"] == 4 + assert profile["tensor_parallelism"] == 2 assert isinstance(approximate["scheduler"], dict) assert isinstance(precise["scheduler"], dict) @@ -187,6 +187,35 @@ def test_release_preset_produces_3_run_specs() -> None: assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) +def test_llama_release_preset_produces_deployment_workload_matrix() -> None: + _init_project_config() + + core_config.project.apply_preset("llama-33-70b-rhoai-release") + + run_specs = runtime_config.get_run_specs() + assert len(run_specs) == 9 + assert {spec.deployment_profile_name for spec in run_specs} == { + "distributed-default", + "precise-prefix-cache", + "approximate-prefix-cache", + } + assert {spec.benchmark_key for spec in run_specs} == { + "concurrent-1k-1k", + "heavy-heterogeneous", + "multi-turn", + } + assert len({spec.namespace for spec in run_specs}) == 9 + + +def test_precise_profile_preserves_kv_events_json_for_the_serving_eval() -> None: + _init_project_config() + core_config.project.set_config("runtime.deployment_profile", "precise-prefix-cache") + + profile = runtime_config.get_deployment_profile() + args = profile["vllm_extra"]["args"] + assert args["kv_events_config"].startswith('\'{"enable_kv_cache_events"') + + def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: variable_overrides_path = env.ARTIFACT_DIR / "000__ci_metadata" / "variable_overrides.yaml" variable_overrides_path.parent.mkdir(parents=True, exist_ok=True) @@ -671,12 +700,12 @@ def test_render_uses_sanitized_model_name_and_profile_resources() -> None: model_cache=runtime_config.get_model_cache_config(), ) - assert manifest["spec"]["replicas"] == 1 + assert manifest["spec"]["replicas"] == 4 assert manifest["spec"]["model"]["uri"] == "hf://openai/gpt-oss-120b" assert manifest["spec"]["model"]["name"] == "openai-gpt-oss-120b" assert manifest["spec"]["template"]["containers"][0]["resources"] == { - "requests": {"nvidia.com/gpu": "1"}, - "limits": {"nvidia.com/gpu": "1"}, + "requests": {"nvidia.com/gpu": "2"}, + "limits": {"nvidia.com/gpu": "2"}, } assert manifest["spec"]["router"]["scheduler"] == {} From f5f94e03f5516e3d4ade3a659ff83e7519807e01 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 27 Jul 2026 08:19:45 +0100 Subject: [PATCH 0349/1072] fix: reuse namespace across release matrix --- projects/llm_d/orchestration/ci.py | 13 ++++++++--- projects/llm_d/orchestration/cleanup_phase.py | 3 +-- .../llm_d/orchestration/config.d/runtime.yaml | 1 - .../orchestration/presets.d/presets.yaml | 1 - .../llm_d/orchestration/runtime_config.py | 23 ++++++++----------- projects/llm_d/tests/test_profiles.py | 9 +++++++- .../toolbox/cleanup_test_resources/main.py | 10 ++++++++ 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index cb0adc76d..dd0858f12 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -24,7 +24,12 @@ ) from projects.core.library.postprocess import run_postprocess_after_test from projects.core.library.replot import caliper_replot_entrypoint -from projects.llm_d.orchestration.cleanup_phase import run as cleanup_toolbox_run +from projects.llm_d.orchestration.cleanup_phase import ( + cleanup_operators, +) +from projects.llm_d.orchestration.cleanup_phase import ( + run as cleanup_toolbox_run, +) from projects.llm_d.orchestration.preflight_phase import run as preflight_toolbox_run from projects.llm_d.orchestration.prepare_sequence import run_prepare_sequence @@ -166,7 +171,8 @@ def pre_cleanup(ctx) -> int: for run_spec in runtime_config.get_run_specs(): with runtime_config.activate_run_spec(run_spec): - cleanup_toolbox_run(namespace=run_spec.namespace, cleanup_subscriptions=True) + cleanup_toolbox_run(namespace=run_spec.namespace) + cleanup_operators() return 0 @@ -180,7 +186,8 @@ def post_cleanup(ctx) -> int: for run_spec in runtime_config.get_run_specs(): with runtime_config.activate_run_spec(run_spec): - cleanup_toolbox_run(namespace=run_spec.namespace, cleanup_subscriptions=True) + cleanup_toolbox_run(namespace=run_spec.namespace) + cleanup_operators() return 0 diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index 1fac3327c..a6ffae6fe 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -28,8 +28,7 @@ def cleanup_namespace(*, namespace: str | None = None) -> None: if namespace is None: namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - inference_service_name = platform["inference_service"]["name"] + inference_service_name = runtime_config.get_inference_service_name() benchmark_name = runtime_config.get_benchmark_job_name() if not oc_resource_exists("namespace", namespace): diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index d092edc41..bea48e83a 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -3,7 +3,6 @@ model_name: Qwen/Qwen3-0.6B deployment_profile: approximate-prefix-cache smoke_request_key: default benchmark_key: null -isolate_run_specs: false job_name: null namespace: forge-llm-d kserve: diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index ce241513d..c0fdfc157 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -48,7 +48,6 @@ gpt-oss-120b-inference-scheduling-release: llama-33-70b-rhoai-release: runtime.selected_preset: llama-33-70b-rhoai-release - runtime.isolate_run_specs: true runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic runtime.deployment_profile: - distributed-default diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f1eb6ba4d..f3b3d9f21 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -12,7 +12,7 @@ import yaml -from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.dsl.utils import slugify_identifier from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -444,6 +444,12 @@ def get_deployment_profile_name() -> str: return deployment_profiles[0] +def get_inference_service_name() -> str: + """Return the rendered LLMInferenceService name for the active profile.""" + base_name = get_platform_config()["inference_service"]["name"] + return f"{base_name}-{get_deployment_profile_name()}" + + def _resolve_template_profile( profile_name: str, deployment_config: dict[str, Any] ) -> dict[str, Any]: @@ -611,25 +617,14 @@ def get_run_specs() -> list[RunSpec]: benchmark_entries = [(None, None)] combinations = list(product(model_names, profile_names, benchmark_entries)) - base_namespace = get_namespace() - isolate_run_specs = bool(_get_runtime_value("isolate_run_specs")) + namespace = get_namespace() run_specs: list[RunSpec] = [] for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) - if not isolate_run_specs or len(combinations) == 1: - namespace = base_namespace - artifact_dirname = f"llmd__{bench_key or 'default'}" - else: - namespace_parts = [base_namespace, model_slug, profile_slug] - artifact_parts = ["llmd", model_slug, profile_slug] - if bench_slug: - namespace_parts.append(bench_slug) - artifact_parts.append(bench_slug) - namespace = truncate_k8s_name("-".join(namespace_parts), max_length=63) - artifact_dirname = "__".join(artifact_parts) + artifact_dirname = f"llmd__{bench_key or 'default'}" run_specs.append( RunSpec( diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 4a17bc923..5a9d3cfd1 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -64,6 +64,13 @@ def test_release_deployment_profiles_have_expected_shape() -> None: assert distributed["scheduler"] == {} +def test_active_profile_inference_service_name_matches_rendered_name() -> None: + _init_project_config() + core_config.project.set_config("runtime.deployment_profile", "precise-prefix-cache") + + assert runtime_config.get_inference_service_name() == "llm-d-precise-prefix-cache" + + @pytest.mark.parametrize( ("preset", "expected_deployment"), [ @@ -204,7 +211,7 @@ def test_llama_release_preset_produces_deployment_workload_matrix() -> None: "heavy-heterogeneous", "multi-turn", } - assert len({spec.namespace for spec in run_specs}) == 9 + assert {spec.namespace for spec in run_specs} == {"forge-llm-d"} def test_precise_profile_preserves_kv_events_json_for_the_serving_eval() -> None: diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 464c9f420..617430798 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -146,6 +146,16 @@ def delete_llm_d_labeled_resources(args, ctx): "forge.openshift.io/project=llm_d", "--ignore-not-found=true", ) + _best_effort_delete( + "llm_d inference services", + "delete", + "llminferenceservice", + "-n", + args.namespace, + "-l", + "forge.openshift.io/project=llm_d", + "--ignore-not-found=true", + ) _best_effort_delete( "llm_d non-preserved pvcs", "delete", From 6705071000763de0b1037d587517674bd97d4c88 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 27 Jul 2026 13:21:52 +0100 Subject: [PATCH 0350/1072] fix: continue release matrix after workload failure --- projects/llm_d/orchestration/ci.py | 1 - .../orchestration/config.d/workloads.yaml | 3 ++ projects/llm_d/tests/test_profiles.py | 43 ++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index dd0858f12..192fd7fc8 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -127,7 +127,6 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" - # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 89a5bffb9..3e41fd3df 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -48,6 +48,9 @@ benchmarks: multi-turn: vllm_args: max_model_len: 40960 + # One Job runs all five rates sequentially; the high-rate cases need more + # than the default one-hour wall-clock budget. + timeout_seconds: 7200 args: backend_type: openai_http rate_type: concurrent diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 5a9d3cfd1..81623eacc 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -2,7 +2,9 @@ import base64 import json +from contextlib import nullcontext from pathlib import Path +from types import SimpleNamespace import pytest import yaml @@ -101,8 +103,9 @@ def test_benchmark_workloads_are_available() -> None: heavy = core_config.project.get_config("workloads.benchmarks.heavy-heterogeneous", print=False) multi_turn = core_config.project.get_config("workloads.benchmarks.multi-turn", print=False) - for benchmark in (short, concurrent, heavy, multi_turn): + for benchmark in (short, concurrent, heavy): assert benchmark["timeout_seconds"] == 3600 + assert multi_turn["timeout_seconds"] == 7200 assert concurrent["args"]["rate"] == [300, 200, 100, 50, 1] assert heavy["args"]["max_seconds"] == 600 @@ -131,7 +134,7 @@ def test_benchmark_resolution_applies_workload_defaults_and_per_benchmark_overri assert multi_turn["job_name"] == "guidellm-benchmark" assert multi_turn["image"] == "ghcr.io/vllm-project/guidellm:v0.5.4" assert multi_turn["pvc_size"] == "1Gi" - assert multi_turn["timeout_seconds"] == 3600 + assert multi_turn["timeout_seconds"] == 7200 def test_guidellm_benchmark_uses_original_model_name_as_processor( @@ -214,6 +217,42 @@ def test_llama_release_preset_produces_deployment_workload_matrix() -> None: assert {spec.namespace for spec in run_specs} == {"forge-llm-d"} +def test_test_matrix_continues_after_a_failed_entry(monkeypatch: pytest.MonkeyPatch) -> None: + _init_project_config() + run_specs = [ + SimpleNamespace( + deployment_profile_name="distributed-default", + benchmark_key="multi-turn", + artifact_dirname="failed-entry", + ), + SimpleNamespace( + deployment_profile_name="precise-prefix-cache", + benchmark_key="multi-turn", + artifact_dirname="next-entry", + ), + ] + calls: list[str] = [] + + monkeypatch.setattr(llmd_ci.runtime_config, "get_run_specs", lambda: run_specs) + monkeypatch.setattr( + llmd_ci.runtime_config, + "activate_run_spec", + lambda _run_spec: nullcontext(), + ) + monkeypatch.setattr(llmd_ci, "trigger_config_review_for_ci", lambda *_args, **_kwargs: None) + + def _test_entry() -> int: + calls.append("run") + if len(calls) == 1: + raise RuntimeError("expected test failure") + return 0 + + monkeypatch.setattr(llmd_ci, "test_toolbox_run", _test_entry) + + assert llmd_ci.run_test_matrix() == 1 + assert calls == ["run", "run"] + + def test_precise_profile_preserves_kv_events_json_for_the_serving_eval() -> None: _init_project_config() core_config.project.set_config("runtime.deployment_profile", "precise-prefix-cache") From e3fdcb090301e27e81668f14a9785d181ea4d365 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 08:52:49 +0100 Subject: [PATCH 0351/1072] fix: keep benchmark timeout state local --- .../tests/test_run_guidellm_benchmark.py | 5 --- .../toolbox/run_guidellm_benchmark/main.py | 43 +++++-------------- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/projects/guidellm/tests/test_run_guidellm_benchmark.py b/projects/guidellm/tests/test_run_guidellm_benchmark.py index 29d4c6b92..8b4f74017 100644 --- a/projects/guidellm/tests/test_run_guidellm_benchmark.py +++ b/projects/guidellm/tests/test_run_guidellm_benchmark.py @@ -3,7 +3,6 @@ import pytest from projects.guidellm.toolbox.run_guidellm_benchmark import build_guidellm_args -from projects.guidellm.toolbox.run_guidellm_benchmark.main import _wait_attempts from projects.guidellm.toolbox.run_guidellm_benchmark.utils import ( expand_guidellm_runs, render_guidellm_job_from_parts, @@ -169,10 +168,6 @@ def test_render_guidellm_job_from_parts_keeps_plain_rates_as_single_guidellm_run ] -def test_wait_attempts_cover_the_job_timeout_and_status_grace_period() -> None: - assert _wait_attempts(3600) == 367 - - def test_build_guidellm_args_renders_list_values() -> None: benchmark = { "outputs": "json", diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index ea4ea0cdc..5457c5487 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -5,12 +5,11 @@ import gzip import json import logging -import math import subprocess import time from pathlib import Path -from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, task from projects.core.dsl.utils import write_json, write_text from projects.core.dsl.utils.k8s import ( oc, @@ -79,20 +78,10 @@ def run( if timeout <= 0: raise ValueError("timeout must be greater than zero") - # Allow time for Kubernetes to record the Job's terminal status after its deadline. - wait_guidellm_benchmark_task._retry_config["attempts"] = _wait_attempts(timeout) execute_tasks(locals()) return 0 -def _wait_attempts(timeout_seconds: int) -> int: - return max( - 1, - math.ceil((timeout_seconds + JOB_COMPLETION_GRACE_SECONDS) / WAIT_POLL_INTERVAL_SECONDS) - + 1, - ) - - @task def validate_parameters(args, ctx): """Validate and normalize parameters""" @@ -208,7 +197,8 @@ def create_guidellm_resources_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" -@retry(attempts=366, delay=WAIT_POLL_INTERVAL_SECONDS, backoff=1.0) +# An upper safety bound only; ctx.wait_deadline enforces the per-run timeout. +@retry(attempts=1080, delay=WAIT_POLL_INTERVAL_SECONDS, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" @@ -229,7 +219,9 @@ def wait_guidellm_benchmark_task(args, ctx): if active: if time.monotonic() >= ctx.wait_deadline: - _raise_benchmark_timeout(args, ctx) + raise TimeoutError( + f"GuideLLM benchmark {ctx.benchmark_name} did not complete within {args.timeout}s" + ) logger.info("Job %s is still active, retrying...", ctx.benchmark_name) return False # Retry immediately @@ -267,13 +259,6 @@ def wait_guidellm_benchmark_task(args, ctx): if succeeded: return f"GuideLLM benchmark {ctx.benchmark_name} completed" if failed: - # Capture state and generate failure file before raising exception - capture_guidellm_state( - artifact_dir=args.artifact_dir, - namespace=ctx.target_namespace, - benchmark_name=ctx.benchmark_name, - ) - # Write failure file failure_file = args.artifact_dir / "FAILURE" failure_message = f"""GuideLLM benchmark job '{ctx.benchmark_name}' failed. @@ -290,21 +275,13 @@ def wait_guidellm_benchmark_task(args, ctx): raise RuntimeError(f"GuideLLM job {ctx.benchmark_name} failed") if time.monotonic() >= ctx.wait_deadline: - _raise_benchmark_timeout(args, ctx) + raise TimeoutError( + f"GuideLLM benchmark {ctx.benchmark_name} did not complete within {args.timeout}s" + ) return False # Retry -def _raise_benchmark_timeout(args, ctx) -> None: - capture_guidellm_state( - artifact_dir=args.artifact_dir, - namespace=ctx.target_namespace, - benchmark_name=ctx.benchmark_name, - ) - raise TimeoutError( - f"GuideLLM benchmark {ctx.benchmark_name} did not complete within {args.timeout}s" - ) - - +@always @task def capture_guidellm_state_task(args, ctx): """Capture GuideLLM benchmark job state and logs""" From 03688e04144dfbb593b738651addfa81065f4dee Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 09:30:36 +0100 Subject: [PATCH 0352/1072] chore: Remove unnecessary comment Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/config.d/model_cache.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/model_cache.yaml b/projects/llm_d/orchestration/config.d/model_cache.yaml index 0c905f127..eae017723 100644 --- a/projects/llm_d/orchestration/config.d/model_cache.yaml +++ b/projects/llm_d/orchestration/config.d/model_cache.yaml @@ -5,8 +5,6 @@ pvc: name_prefix: llm-d-model size: 15Gi access_mode: ReadWriteOnce - # GitHub /var overrides can select an RWX-capable class together with - # access_mode: ReadWriteMany for multi-node model-cache mounts. storage_class_name: null model_directory_name: model From 9ce4e080585b62be04bbb1955bf846750b71f0cc Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 11:38:40 +0100 Subject: [PATCH 0353/1072] fix: resolve release matrix rebase regressions --- projects/llm_d/orchestration/config.d/deployments.yaml | 1 - projects/llm_d/orchestration/config.d/workloads.yaml | 1 + projects/llm_d/orchestration/render_inference_service.py | 5 ++++- projects/llm_d/orchestration/runtime_config.py | 2 +- projects/llm_d/tests/test_profiles.py | 9 ++++----- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 70e26ede2..7d71d1cdc 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -6,7 +6,6 @@ defaults: tensor_parallelism: 1 vllm_extra: args: - max_model_len: 4096 gpu_memory_utilization: 0.92 trust_remote_code: true no_enable_log_requests: true diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 3e41fd3df..09bed7c4b 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -36,6 +36,7 @@ benchmarks: max_seconds: 600 heavy-heterogeneous: + timeout_seconds: 3600 vllm_args: max_model_len: 40960 args: diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index 819452e32..6e8770f02 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -258,7 +258,10 @@ def _build_vllm_additional_args( # Add workload vllm_args if available if workload and "vllm_args" in workload: workload_vllm_args = _build_vllm_args(workload["vllm_args"]) - vllm_deploy_args.extend(workload_vllm_args) + deployment_options = {arg.split("=", 1)[0] for arg in vllm_deploy_args} + vllm_deploy_args.extend( + arg for arg in workload_vllm_args if arg.split("=", 1)[0] not in deployment_options + ) return " ".join(vllm_deploy_args) diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index f3b3d9f21..ab2b8f72a 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -624,7 +624,7 @@ def get_run_specs() -> list[RunSpec]: for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) - artifact_dirname = f"llmd__{bench_key or 'default'}" + artifact_dirname = f"llmd__{bench_key or 'default'}__{profile_slug}" run_specs.append( RunSpec( diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 81623eacc..783ccbe30 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -233,13 +233,12 @@ def test_test_matrix_continues_after_a_failed_entry(monkeypatch: pytest.MonkeyPa ] calls: list[str] = [] - monkeypatch.setattr(llmd_ci.runtime_config, "get_run_specs", lambda: run_specs) + monkeypatch.setattr(test_phase.runtime_config, "get_run_specs", lambda: run_specs) monkeypatch.setattr( - llmd_ci.runtime_config, + test_phase.runtime_config, "activate_run_spec", lambda _run_spec: nullcontext(), ) - monkeypatch.setattr(llmd_ci, "trigger_config_review_for_ci", lambda *_args, **_kwargs: None) def _test_entry() -> int: calls.append("run") @@ -247,9 +246,9 @@ def _test_entry() -> int: raise RuntimeError("expected test failure") return 0 - monkeypatch.setattr(llmd_ci, "test_toolbox_run", _test_entry) + monkeypatch.setattr(test_phase, "do_test", _test_entry) - assert llmd_ci.run_test_matrix() == 1 + assert test_phase.run_all_tests(stop_on_error=False) == 1 assert calls == ["run", "run"] From 5080b5ed307089575af6c90f6c75bc742b274526 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 13:57:52 +0300 Subject: [PATCH 0354/1072] [caliper] kpi: generic kpis.json to metrics.json conversion for MLflow Move metrics.json generation from project-specific parsers into caliper as a generic mechanism. After kpis.json is produced, caliper now automatically writes per-run metrics.json and parameters.json files using KPI ids as metric keys. The MLflow export backend picks these up via _log_metrics_and_params_from_tree without any project-specific code. - Add caliper/engine/kpi/metrics_from_kpis.py with generate_metrics_from_kpis() - Wire into postprocess pipeline after kpis.json generation step - Remove mcp_gateway-specific metrics.json/parameters.json writing - Simplify mcp_gateway notifications MLflow metric key lookup - Update tests to reflect new ownership Co-authored-by: Cursor --- .../caliper/engine/kpi/metrics_from_kpis.py | 144 ++++++++++++++++++ projects/caliper/orchestration/postprocess.py | 36 +++++ .../orchestration/notifications.py | 17 +-- .../mcp_gateway/parsing/parsers.py | 18 +-- .../tests/test_mcp_gateway_plugin.py | 27 +--- 5 files changed, 190 insertions(+), 52 deletions(-) create mode 100644 projects/caliper/engine/kpi/metrics_from_kpis.py diff --git a/projects/caliper/engine/kpi/metrics_from_kpis.py b/projects/caliper/engine/kpi/metrics_from_kpis.py new file mode 100644 index 000000000..408ae4981 --- /dev/null +++ b/projects/caliper/engine/kpi/metrics_from_kpis.py @@ -0,0 +1,144 @@ +"""Generic kpis.json -> metrics.json + parameters.json conversion. + +Reads a hierarchical kpis.json (schema v2) and writes per-test-run +metrics.json and parameters.json files into the matching artifact tree +directories. The MLflow export backend picks these up automatically via +``_log_metrics_and_params_from_tree``. + +This replaces project-specific metrics.json generation (e.g. in +mcp_gateway parsers) with a single generic caliper mechanism that works +for every project producing a kpis.json. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" + + +def _build_run_dir_index(artifact_tree: Path) -> dict[str, Path]: + """Map run directory names to their paths using __test_labels__.yaml markers.""" + index: dict[str, Path] = {} + for marker in sorted(artifact_tree.rglob(TEST_LABELS_MARKER)): + if marker.is_file(): + run_dir = marker.parent + try: + rel = run_dir.relative_to(artifact_tree) + except ValueError: + rel = Path(run_dir.name) + index[str(rel)] = run_dir + index[run_dir.name] = run_dir + return index + + +def _is_scalar(value: Any) -> bool: + """Check if a KPI value is a scalar number (not 2D data).""" + return isinstance(value, int | float) and not isinstance(value, bool) + + +def _write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + + +def generate_metrics_from_kpis( + kpis_json_path: Path, + artifact_tree: Path, +) -> dict[str, Any]: + """Convert kpis.json into per-run metrics.json and parameters.json files. + + For each test entry in kpis.json, finds the matching directory under + ``artifact_tree`` (via ``__test_labels__.yaml`` markers) and writes: + + - ``metrics.json``: ``{kpi_id: value}`` for all scalar KPIs + - ``parameters.json``: test-level labels as string key-value pairs + + Args: + kpis_json_path: Path to the kpis.json file (schema v2). + artifact_tree: Root of the caliper artifact tree containing + test run directories with ``__test_labels__.yaml`` markers. + + Returns: + Status dict with counts and any warnings. + """ + if not kpis_json_path.is_file(): + return {"status": "skipped", "reason": f"kpis.json not found: {kpis_json_path}"} + + try: + with kpis_json_path.open(encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to read kpis.json at %s: %s", kpis_json_path, e) + return {"status": "failed", "error": str(e)} + + if not isinstance(data, dict) or data.get("schema_version") != "2": + return {"status": "skipped", "reason": "Not a schema v2 kpis.json"} + + tests = data.get("tests", []) + if not tests: + return {"status": "skipped", "reason": "No tests in kpis.json"} + + run_dir_index = _build_run_dir_index(artifact_tree) + if not run_dir_index: + logger.warning("No test run directories found under %s", artifact_tree) + return {"status": "skipped", "reason": "No run directories with __test_labels__.yaml found"} + + written = 0 + warnings: list[str] = [] + + for test_entry in tests: + run_id = test_entry.get("run_id", "") + test_base_path = ( + test_entry.get("metadata", {}).get("source", {}).get("test_base_path", run_id) + ) + + run_dir = run_dir_index.get(test_base_path) or run_dir_index.get(run_id) + if run_dir is None: + warnings.append(f"No matching directory for run_id={run_id!r}") + continue + + kpis = test_entry.get("kpis", []) + metrics: dict[str, float] = {} + for kpi in kpis: + kpi_id = kpi.get("id", "") + value = kpi.get("value") + if kpi_id and _is_scalar(value): + metrics[kpi_id] = value + + if metrics: + _write_json(run_dir / METRICS_FILE, metrics) + + labels = test_entry.get("labels", {}) + if labels: + params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} + _write_json(run_dir / PARAMETERS_FILE, params) + + written += 1 + + result: dict[str, Any] = { + "status": "success", + "tests_processed": written, + "total_tests": len(tests), + } + if warnings: + result["warnings"] = warnings + for w in warnings: + logger.warning("kpis-to-metrics: %s", w) + + logger.info( + "Generated metrics.json for %d/%d test(s) from %s", + written, + len(tests), + kpis_json_path.name, + ) + return result diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 9fd7b1927..41f1306af 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1001,6 +1001,9 @@ def _run_kpi_and_ai_data_steps(self) -> None: # KPI JSON generation self._run_artifacts_to_kpis_step(output_dir, mod_str) + # Generate per-run metrics.json + parameters.json from kpis.json + self._run_kpis_to_metrics_step(output_dir) + # KPI CSV export self._run_kpis_to_csv_step(output_dir) @@ -1040,6 +1043,39 @@ def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: }, ) + def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: + """Generate per-run metrics.json + parameters.json from kpis.json. + + Runs automatically after kpis.json generation succeeds. Writes + files into each test run directory so the MLflow export backend + picks them up via ``_log_metrics_and_params_from_tree``. + """ + kpi_step = self._get_step("artifacts_to_kpis") + if not kpi_step or kpi_step.get("status") != "success": + return + + kpis_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + if not kpis_json_path.is_file(): + logger.warning("kpis.json not found at %s, skipping metrics generation", kpis_json_path) + return + + try: + from projects.caliper.engine.kpi.metrics_from_kpis import generate_metrics_from_kpis + + result = generate_metrics_from_kpis(kpis_json_path, self.tree_root) + status = result.get("status", "unknown") + if status == "success": + logger.info( + "kpis-to-metrics: wrote metrics.json for %d test(s)", + result.get("tests_processed", 0), + ) + elif status == "skipped": + logger.info("kpis-to-metrics: skipped — %s", result.get("reason", "")) + else: + logger.warning("kpis-to-metrics: %s", result.get("error", "unknown error")) + except Exception as e: + logger.warning("kpis-to-metrics conversion failed (non-fatal): %s", e) + def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" if not self.config.kpi.kpis_to_csv.enabled: diff --git a/projects/mcp_gateway/orchestration/notifications.py b/projects/mcp_gateway/orchestration/notifications.py index 297eb9aa8..0c4683960 100644 --- a/projects/mcp_gateway/orchestration/notifications.py +++ b/projects/mcp_gateway/orchestration/notifications.py @@ -37,13 +37,9 @@ ("mcp_gw_failure_rate", "Failure rate", "%"), ] -# Mapping from kpis.jsonl IDs to MLflow metrics.json keys (logged via mlflow.log_metric) -_KPI_TO_MLFLOW_METRIC = { - "mcp_gw_requests_per_second": "requests_per_second", - "mcp_gw_p95_ms": "p95_ms", - "mcp_gw_p99_ms": "p99_ms", - "mcp_gw_failure_rate": "failure_rate", -} +# KPI IDs are now logged directly as MLflow metric keys via the generic +# caliper kpis-to-metrics conversion — no mapping needed. +_KPI_IDS = {kpi_id for kpi_id, _, _ in TARGET_KPIS} _RUN_NAME_PREFIX = "forge-mcp-gateway-" @@ -376,12 +372,7 @@ def _load_previous_kpis_from_mlflow( return {}, "", True raw_metrics = previous_run.data.metrics or {} - mlflow_to_kpi = {v: k for k, v in _KPI_TO_MLFLOW_METRIC.items()} - metrics = {} - for mlflow_key, value in raw_metrics.items(): - kpi_id = mlflow_to_kpi.get(mlflow_key) - if kpi_id: - metrics[kpi_id] = value + metrics = {k: v for k, v in raw_metrics.items() if k in _KPI_IDS} return metrics, prev_name, False diff --git a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py index 107f60ae5..6fd9e2cdf 100644 --- a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py +++ b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py @@ -2,9 +2,7 @@ from __future__ import annotations -import json import logging -from pathlib import Path from typing import Any from projects.agentic_tools.locust.helpers.parse_results import RunMetrics, parse_stats_csv @@ -17,8 +15,6 @@ logger = logging.getLogger(__name__) STATS_CSV = "stats.csv" -METRICS_FILE = "metrics.json" -PARAMETERS_FILE = "parameters.json" def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: @@ -33,7 +29,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: - """Convert RunMetrics to a flat dictionary suitable for metrics.json.""" + """Convert RunMetrics to a flat dictionary for unified result records.""" return { "total_requests": metrics.total_requests, "total_failures": metrics.total_failures, @@ -48,13 +44,6 @@ def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: } -def _write_json(path: Path, data: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write("\n") - - class MCPGatewayParser: """Parser for Locust stats.csv artifacts from MCP Gateway tests.""" @@ -89,11 +78,6 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: labels = _labels_from_node(node) metrics_dict = _run_metrics_to_dict(run_metrics) - _write_json(node.directory / METRICS_FILE, metrics_dict) - - params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} - _write_json(node.directory / PARAMETERS_FILE, params) - records.append( UnifiedResultRecord( test_base_path=str(node.test_path), diff --git a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py index 78f45ab95..d45287aab 100644 --- a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py +++ b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path import pytest @@ -66,30 +65,14 @@ def test_parse_creates_records(self, tmp_path: Path): assert record.metrics["requests_per_second"] == 31.5 assert record.metrics["p95_ms"] == 80.0 - def test_parse_writes_metrics_json(self, tmp_path: Path): + def test_parse_does_not_write_metrics_json(self, tmp_path: Path): + """Parser no longer writes metrics.json — caliper handles it generically.""" node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) parser = MCPGatewayParser() parser.parse(tmp_path, [node]) - metrics_file = tmp_path / "run-a" / "metrics.json" - assert metrics_file.exists() - data = json.loads(metrics_file.read_text()) - assert data["total_requests"] == 1000 - assert data["requests_per_second"] == 31.5 - - def test_parse_writes_parameters_json(self, tmp_path: Path): - node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) - parser = MCPGatewayParser() - - parser.parse(tmp_path, [node]) - - params_file = tmp_path / "run-a" / "parameters.json" - assert params_file.exists() - data = json.loads(params_file.read_text()) - assert data["preset"] == "smoke" - assert data["target"] == "gateway" - assert data["users"] == "16" + assert not (tmp_path / "run-a" / "metrics.json").exists() def test_parse_no_stats_csv(self, tmp_path: Path): node_dir = tmp_path / "run-empty" @@ -123,8 +106,8 @@ def test_parse_multiple_nodes(self, tmp_path: Path): assert "run-b" in paths for node_name in ["run-a", "run-b"]: - assert (tmp_path / node_name / "metrics.json").exists() - assert (tmp_path / node_name / "parameters.json").exists() + assert not (tmp_path / node_name / "metrics.json").exists() + assert not (tmp_path / node_name / "parameters.json").exists() # --------------------------------------------------------------------------- From e008f55290bb99ac6e14a078663853fba2dd9045 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:39:09 +0200 Subject: [PATCH 0355/1072] [caliper] cli: commands: include the workspace in the artifacts export CLI --- projects/caliper/cli/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 849d2f0d7..2ccce1951 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -956,6 +956,7 @@ def kpi_s3_import( @click.option("--mlflow-experiment", default=None, envvar="MLFLOW_EXPERIMENT_NAME") @click.option("--mlflow-run-id", default=None, envvar="MLFLOW_RUN_ID") @click.option("--mlflow-run-name", default=None, envvar="CALIPER_MLFLOW_RUN_NAME") +@click.option("--mlflow-workspace", default=None, help="MLflow workspace name") @click.option("--mlflow-insecure-tls", is_flag=True) @click.option( "--mlflow-secrets", @@ -988,6 +989,7 @@ def artifacts_export( mlflow_experiment: str | None, mlflow_run_id: str | None, mlflow_run_name: str | None, + mlflow_workspace: str | None, mlflow_insecure_tls: bool, mlflow_secrets_path: Path | None, mlflow_config_path: Path | None, @@ -996,6 +998,7 @@ def artifacts_export( status_yaml_path: Path | None, upload_workers: int, ) -> None: + # Load config from file if provided, CLI args override config = {} if mlflow_config_path: @@ -1011,6 +1014,7 @@ def artifacts_export( "experiment": mlflow_experiment or config.get("experiment"), "run_id": mlflow_run_id or config.get("run_id"), "run_name": mlflow_run_name or config.get("run_name"), + "workspace": mlflow_workspace or config.get("workspace"), "insecure_tls": mlflow_insecure_tls or config.get("insecure_tls", False), "secrets_path": mlflow_secrets_path or config.get("secrets_path"), "upload_workers": upload_workers, From 60523345e0d417c310541c5f960fc8e8ada1040c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:39:34 +0200 Subject: [PATCH 0356/1072] [caliper] cli: commands: fix wrong argname --- projects/caliper/cli/commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 2ccce1951..9dba9550a 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -1027,10 +1027,15 @@ def artifacts_export( dry_run=dry_run, verbose=verbose, status_yaml_path=status_yaml_path, - mlflow_config=final_config, + mlflow_config_data=final_config, ) except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() click.echo(f"artifacts export failed: {e}", err=True) + click.echo("Full traceback:", err=True) + click.echo(full_traceback, err=True) sys.exit(3) From 8ccf6a36280c3d26d27f62f6d06df6f213fdc8e2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:42:03 +0200 Subject: [PATCH 0357/1072] [caliper] engine: file_export: artifacts_export_run: wrong argname --- projects/caliper/engine/file_export/artifacts_export_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index 2dad7bf54..464fb4843 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -192,7 +192,7 @@ def run_artifacts_export( mlflow_tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") if not mlflow_tracking_uri: click.echo( - "MLflow backend requires a tracking URI: --mlflow-endpoint / MLFLOW_TRACKING_URI, " + "MLflow backend requires a tracking URI: --mlflow-tracking-uri / MLFLOW_TRACKING_URI, " "or tracking_uri in --mlflow-secrets or --mlflow-config.", err=True, ) From 2d8344aec54f1bf30f487dabf33e95cf57bd6adc Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 15:28:44 +0100 Subject: [PATCH 0358/1072] fix: keep smoke profiles at default sizing --- .../orchestration/config.d/deployments.yaml | 13 +++++++++++-- .../llm_d/orchestration/presets.d/presets.yaml | 6 +++--- projects/llm_d/tests/test_profiles.py | 18 +++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index 7d71d1cdc..89b471010 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -28,13 +28,22 @@ pd: profiles: approximate-prefix-cache: scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml + + precise-prefix-cache: + scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml + + distributed-default: + scheduler: {} + + release-approximate-prefix-cache: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml replicas: 4 tensor_parallelism: 2 vllm_extra: args: max_model_len: 40960 - precise-prefix-cache: + release-precise-prefix-cache: scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml replicas: 4 tensor_parallelism: 2 @@ -59,7 +68,7 @@ profiles: - name: PYTHONHASHSEED value: "42" - distributed-default: + release-distributed-default: scheduler: {} replicas: 4 tensor_parallelism: 2 diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index c0fdfc157..796a3c66e 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -50,9 +50,9 @@ llama-33-70b-rhoai-release: runtime.selected_preset: llama-33-70b-rhoai-release runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic runtime.deployment_profile: - - distributed-default - - precise-prefix-cache - - approximate-prefix-cache + - release-distributed-default + - release-precise-prefix-cache + - release-approximate-prefix-cache runtime.smoke_request_key: default runtime.benchmark_key: - concurrent-1k-1k diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index 783ccbe30..d6e07417d 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -50,11 +50,11 @@ def test_deployment_presets_resolve_deployments() -> None: def test_release_deployment_profiles_have_expected_shape() -> None: _init_project_config() - core_config.project.set_config("runtime.deployment_profile", "approximate-prefix-cache") + core_config.project.set_config("runtime.deployment_profile", "release-approximate-prefix-cache") approximate = runtime_config.get_deployment_profile() - core_config.project.set_config("runtime.deployment_profile", "precise-prefix-cache") + core_config.project.set_config("runtime.deployment_profile", "release-precise-prefix-cache") precise = runtime_config.get_deployment_profile() - core_config.project.set_config("runtime.deployment_profile", "distributed-default") + core_config.project.set_config("runtime.deployment_profile", "release-distributed-default") distributed = runtime_config.get_deployment_profile() for profile in (approximate, precise, distributed): @@ -142,7 +142,7 @@ def test_guidellm_benchmark_uses_original_model_name_as_processor( ) -> None: _init_project_config() core_config.project.set_config("runtime.model_name", "openai/gpt-oss-120b") - core_config.project.set_config("runtime.deployment_profile", "distributed-default") + core_config.project.set_config("runtime.deployment_profile", "release-distributed-default") core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") captured: dict[str, object] = {} @@ -205,9 +205,9 @@ def test_llama_release_preset_produces_deployment_workload_matrix() -> None: run_specs = runtime_config.get_run_specs() assert len(run_specs) == 9 assert {spec.deployment_profile_name for spec in run_specs} == { - "distributed-default", - "precise-prefix-cache", - "approximate-prefix-cache", + "release-distributed-default", + "release-precise-prefix-cache", + "release-approximate-prefix-cache", } assert {spec.benchmark_key for spec in run_specs} == { "concurrent-1k-1k", @@ -254,7 +254,7 @@ def _test_entry() -> int: def test_precise_profile_preserves_kv_events_json_for_the_serving_eval() -> None: _init_project_config() - core_config.project.set_config("runtime.deployment_profile", "precise-prefix-cache") + core_config.project.set_config("runtime.deployment_profile", "release-precise-prefix-cache") profile = runtime_config.get_deployment_profile() args = profile["vllm_extra"]["args"] @@ -733,7 +733,7 @@ def test_render_uses_sanitized_model_name_and_profile_resources() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) core_config.project.set_config("runtime.model_name", "openai/gpt-oss-120b") - core_config.project.set_config("runtime.deployment_profile", "distributed-default") + core_config.project.set_config("runtime.deployment_profile", "release-distributed-default") manifest = render_inference_service_from_parts( config_dir=str(PROJECT_ORCHESTRATION_DIR), From 28a08cb94693d2bbedf33c8e6efedf0ca1c84ad3 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 16:52:14 +0100 Subject: [PATCH 0359/1072] feat: add cpt release testing presets --- .../llm_d/orchestration/presets.d/cpt.yaml | 29 +++++++++++++++++ .../orchestration/presets.d/presets.yaml | 25 --------------- projects/llm_d/tests/test_profiles.py | 32 ++++++++++++++++--- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index ce7ce6859..db69ceed1 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -11,3 +11,32 @@ cpt-reference-flavors: cpt: caliper.export.backend.mlflow.config.experiment: cpt-llm-d + cpt.kpi.labels.test_harness: rhoai-release + +cpt-release-testing-gpt-oss-120b: + extends: + - cpt + - gpt-oss + runtime.benchmark_key: + - concurrent-1k-1k + - heavy-heterogeneous + - multi-turn + +cpt-release-testing-llama-33-70b: + extends: + - cpt + runtime.selected_preset: cpt-release-testing-llama-33-70b + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic + runtime.deployment_profile: + - release-distributed-default + - release-precise-prefix-cache + - release-approximate-prefix-cache + runtime.smoke_request_key: default + runtime.benchmark_key: + - concurrent-1k-1k + - heavy-heterogeneous + - multi-turn + model_cache.pvc.size: 300Gi + platform.operators.rhods-operator.channel: beta + workloads.args: + request_type: text_completions diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 796a3c66e..393f55e0b 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -37,28 +37,3 @@ benchmark-short: runtime.model_name: meta-llama/Llama-3.1-8B-Instruct runtime.smoke_request_key: default runtime.benchmark_key: short - -gpt-oss-120b-inference-scheduling-release: - extends: - - gpt-oss - runtime.benchmark_key: - - concurrent-1k-1k - - heavy-heterogeneous - - multi-turn - -llama-33-70b-rhoai-release: - runtime.selected_preset: llama-33-70b-rhoai-release - runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - runtime.deployment_profile: - - release-distributed-default - - release-precise-prefix-cache - - release-approximate-prefix-cache - runtime.smoke_request_key: default - runtime.benchmark_key: - - concurrent-1k-1k - - heavy-heterogeneous - - multi-turn - model_cache.pvc.size: 300Gi - platform.operators.rhods-operator.channel: beta - workloads.args: - request_type: text_completions diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index d6e07417d..e7c88ee35 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -163,7 +163,18 @@ def _fake_run(**kwargs): def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> None: _init_project_config() - core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + core_config.project.apply_preset("cpt-release-testing-gpt-oss-120b") + + assert ( + core_config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", print=False + ) + == "cpt-llm-d" + ) + assert ( + core_config.project.get_config("cpt.kpi.labels.test_harness", print=False) + == "rhoai-release" + ) assert runtime_config.get_deployment_profile_name() == "distributed-default" assert runtime_config.get_model_cache_config()["pvc"]["size"] == "300Gi" @@ -183,7 +194,7 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non def test_release_preset_produces_3_run_specs() -> None: _init_project_config() - core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + core_config.project.apply_preset("cpt-release-testing-gpt-oss-120b") run_specs = runtime_config.get_run_specs() @@ -200,7 +211,18 @@ def test_release_preset_produces_3_run_specs() -> None: def test_llama_release_preset_produces_deployment_workload_matrix() -> None: _init_project_config() - core_config.project.apply_preset("llama-33-70b-rhoai-release") + core_config.project.apply_preset("cpt-release-testing-llama-33-70b") + + assert ( + core_config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", print=False + ) + == "cpt-llm-d" + ) + assert ( + core_config.project.get_config("cpt.kpi.labels.test_harness", print=False) + == "rhoai-release" + ) run_specs = runtime_config.get_run_specs() assert len(run_specs) == 9 @@ -267,7 +289,7 @@ def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> variable_overrides_path.write_text( yaml.safe_dump( { - "project.args": ["gpt-oss-120b-inference-scheduling-release"], + "project.args": ["cpt-release-testing-gpt-oss-120b"], "runtime.benchmark_key": "multi-turn", }, sort_keys=True, @@ -804,7 +826,7 @@ def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> N def test_benchmark_job_name_from_activated_spec() -> None: _init_project_config() - core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + core_config.project.apply_preset("cpt-release-testing-gpt-oss-120b") for run_spec in runtime_config.get_run_specs(): with runtime_config.activate_run_spec(run_spec): From a157ca18c7879e706a545e265b2fdbf764540669 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 17:04:05 +0100 Subject: [PATCH 0360/1072] fix: align gpt oss release matrix with llama --- .../llm_d/orchestration/presets.d/cpt.yaml | 27 +++++++++---------- projects/llm_d/tests/test_profiles.py | 26 +++++++++++++----- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index db69ceed1..029603033 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -13,20 +13,9 @@ cpt: caliper.export.backend.mlflow.config.experiment: cpt-llm-d cpt.kpi.labels.test_harness: rhoai-release -cpt-release-testing-gpt-oss-120b: +cpt-release-testing-rhoai: extends: - cpt - - gpt-oss - runtime.benchmark_key: - - concurrent-1k-1k - - heavy-heterogeneous - - multi-turn - -cpt-release-testing-llama-33-70b: - extends: - - cpt - runtime.selected_preset: cpt-release-testing-llama-33-70b - runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic runtime.deployment_profile: - release-distributed-default - release-precise-prefix-cache @@ -38,5 +27,15 @@ cpt-release-testing-llama-33-70b: - multi-turn model_cache.pvc.size: 300Gi platform.operators.rhods-operator.channel: beta - workloads.args: - request_type: text_completions + +cpt-release-testing-gpt-oss-120b: + extends: + - gpt-oss + - cpt-release-testing-rhoai + runtime.selected_preset: cpt-release-testing-gpt-oss-120b + +cpt-release-testing-llama-33-70b: + extends: + - cpt-release-testing-rhoai + runtime.selected_preset: cpt-release-testing-llama-33-70b + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index e7c88ee35..f1473cd21 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -176,7 +176,11 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non == "rhoai-release" ) - assert runtime_config.get_deployment_profile_name() == "distributed-default" + assert core_config.project.get_config("runtime.deployment_profile", print=False) == [ + "release-distributed-default", + "release-precise-prefix-cache", + "release-approximate-prefix-cache", + ] assert runtime_config.get_model_cache_config()["pvc"]["size"] == "300Gi" assert runtime_config.get_benchmark_keys() == [ "concurrent-1k-1k", @@ -191,21 +195,25 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non assert benchmark["args"]["request_type"] == "text_completions" -def test_release_preset_produces_3_run_specs() -> None: +def test_gpt_release_preset_produces_deployment_workload_matrix() -> None: _init_project_config() core_config.project.apply_preset("cpt-release-testing-gpt-oss-120b") run_specs = runtime_config.get_run_specs() - assert len(run_specs) == 3 - assert [spec.benchmark_key for spec in run_specs] == [ + assert len(run_specs) == 9 + assert {spec.benchmark_key for spec in run_specs} == { "concurrent-1k-1k", "heavy-heterogeneous", "multi-turn", - ] + } assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) - assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) + assert {spec.deployment_profile_name for spec in run_specs} == { + "release-distributed-default", + "release-precise-prefix-cache", + "release-approximate-prefix-cache", + } def test_llama_release_preset_produces_deployment_workload_matrix() -> None: @@ -300,7 +308,11 @@ def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> llmd_ci.init() assert runtime_config.get_model_name() == "openai/gpt-oss-120b" - assert runtime_config.get_deployment_profile_name() == "distributed-default" + assert core_config.project.get_config("runtime.deployment_profile", print=False) == [ + "release-distributed-default", + "release-precise-prefix-cache", + "release-approximate-prefix-cache", + ] assert runtime_config.get_benchmark_keys() == ["multi-turn"] From c01f6422e0d41c3cbede1bfebbb6c14b503d31e8 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 3 Aug 2026 17:09:17 +0100 Subject: [PATCH 0361/1072] refactor: reuse llama release model preset --- projects/llm_d/orchestration/presets.d/cpt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index 029603033..c8924f89d 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -36,6 +36,6 @@ cpt-release-testing-gpt-oss-120b: cpt-release-testing-llama-33-70b: extends: + - llama3-70b - cpt-release-testing-rhoai runtime.selected_preset: cpt-release-testing-llama-33-70b - runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic From 2eba2c92c5e8385329adf45beb56cb25849463be Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 2 Aug 2026 15:27:16 +0300 Subject: [PATCH 0362/1072] fix: layering issue in caliper, moved the export multi-run expiriment export to core and orchestration is importing it --- .../file_export/artifacts_export_run.py | 161 +++++++++++++++++ projects/caliper/orchestration/export.py | 170 ++---------------- .../caliper/tests/test_multi_run_export.py | 40 ++--- 3 files changed, 197 insertions(+), 174 deletions(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index 2dad7bf54..8607c9291 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -252,3 +252,164 @@ def run_artifacts_export( if any(r.status == "failure" for r in results): return 4 return 0 + + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" + + +def discover_run_dirs(from_path: Path) -> list[Path]: + """Auto-detect test run directories via ``__test_labels__.yaml`` markers.""" + run_dirs: list[Path] = [] + for marker in sorted(from_path.rglob(TEST_LABELS_MARKER)): + if marker.is_file(): + run_dirs.append(marker.parent) + return run_dirs + + +def run_multi_run_artifacts_export( + *, + from_path: Path, + run_dirs: list[Path], + backend: tuple[str, ...] | list[str], + mlflow_experiment: str | None = None, + mlflow_run_name: str | None = None, + mlflow_secrets_path: Path | None = None, + mlflow_config_data: dict[str, Any] | None = None, + child_run_names: dict[Path, str] | None = None, + verbose: bool = False, + status_yaml_path: Path | None = None, + upload_workers: int = 10, +) -> int: + """Run multi-run file export (parent + nested child MLflow runs). + + Mirrors :func:`run_artifacts_export` for the multi-run case. + Returns a process exit code (0 = success, 1 = config error, 4 = export failure). + """ + from projects.caliper.engine.file_export import mlflow_backend + + backends = [b.strip().lower() for b in backend if b.strip()] + if not backends: + click.echo("Specify at least one --backend.", err=True) + return 1 + + if "mlflow" not in backends: + click.echo( + f"only 'mlflow' backend export is supported for multi-run " + f"(got '{' '.join(backends)}').", + err=True, + ) + return 1 + + secrets_data: dict[str, Any] | None = None + config_data: dict[str, Any] | None = None + if mlflow_secrets_path is not None: + try: + secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) + validate_mlflow_secrets(secrets_data) + except (OSError, ValueError, TypeError, yaml.YAMLError) as e: + click.echo(f"Invalid MLflow secrets file: {e}", err=True) + return 1 + if mlflow_config_data is not None: + try: + config_data = dict(mlflow_config_data) + validate_mlflow_config(config_data) + except (ValueError, TypeError) as e: + click.echo(f"Invalid MLflow settings dict: {e}", err=True) + return 1 + + merged_ml = merge_mlflow_files_with_cli( + None, + secrets_data=secrets_data, + config_data=config_data, + cli_tracking_uri=None, + cli_experiment=mlflow_experiment, + cli_run_id=None, + cli_run_name=mlflow_run_name, + ) + + secret_part = project_secrets_fields(merged_ml) + mlflow_connection = secret_part if secret_part else None + + tracking_uri = merged_ml.get("tracking_uri") + experiment = merged_ml.get("experiment") + run_name = mlflow_run_name or merged_ml.get("run_name") + workspace = merged_ml.get("workspace") + if not workspace: + click.echo("The export workspace must be specified", err=True) + return 1 + + meta = project_metadata_fields(merged_ml) + run_metadata = meta if meta else None + + insecure_tls = bool(mlflow_connection and mlflow_connection.get("insecure_tls")) + + if not tracking_uri: + tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") + if not tracking_uri: + click.echo("MLflow backend requires a tracking URI.", err=True) + return 1 + try: + assert_tracking_uri_has_no_userinfo(tracking_uri) + except ValueError as e: + click.echo(f"Invalid MLflow tracking URI: {e}", err=True) + return 1 + + all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] + + if verbose: + click.echo("caliper multi-run export (verbose)", err=True) + click.echo(f" Source: {from_path}", err=True) + click.echo(f" Total artifact files: {len(all_artifact_paths)}", err=True) + click.echo(f" Run directories: {len(run_dirs)}", err=True) + click.echo(f" Workspace: {workspace}", err=True) + for rd in run_dirs: + click.echo(f" - {rd.name}", err=True) + click.echo("", err=True) + + try: + detail, ml_meta = mlflow_backend.log_multi_run_artifacts( + all_artifact_paths=all_artifact_paths, + artifact_root=from_path, + run_dirs=run_dirs, + metrics_file=METRICS_FILE, + parameters_file=PARAMETERS_FILE, + tracking_uri=tracking_uri, + experiment=experiment, + parent_run_name=run_name, + insecure_tls=insecure_tls, + connection=mlflow_connection, + verbose=verbose, + upload_workers=upload_workers, + run_metadata=run_metadata, + workspace=workspace, + child_run_names=child_run_names or None, + ) + results = [ + FileExportBackendResult( + backend="mlflow", + status="success", + detail=detail, + metadata=ml_meta, + ) + ] + except Exception as e: + traceback.print_exception(e, file=sys.stderr) + click.echo(f"multi-run export failed: {e}", err=True) + results = [FileExportBackendResult(backend="mlflow", status="failure", detail=str(e))] + + for r in results: + click.echo(f"{r.backend}: {r.status} {r.detail}") + + if status_yaml_path is not None: + try: + write_artifacts_status_yaml(status_yaml_path, results) + click.echo(f"Wrote status YAML to {status_yaml_path}") + except OSError as e: + click.echo(f"Failed to write status YAML ({status_yaml_path}): {e}", err=True) + return 4 + + if any(r.status == "failure" for r in results): + return 4 + return 0 diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index b7b066c63..59a0f4c96 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -2,8 +2,8 @@ Config-driven Caliper artifact export for FORGE orchestration projects (e.g. skeleton). Validates :class:`~projects.caliper.orchestration.export_config.CaliperOrchestrationExportConfig` -and calls -:func:`projects.caliper.engine.file_export.artifacts_export_run.run_artifacts_export`. +and calls :func:`projects.caliper.engine.file_export.run_artifacts_export` / +:func:`projects.caliper.engine.file_export.run_multi_run_artifacts_export`. """ from __future__ import annotations @@ -18,11 +18,12 @@ import yaml from pydantic import ValidationError -from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export -from projects.caliper.engine.file_export.mlflow_config import ( - load_mlflow_config_yaml, - project_metadata_fields, +from projects.caliper.engine.file_export.artifacts_export_run import ( + discover_run_dirs, + run_artifacts_export, + run_multi_run_artifacts_export, ) +from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml from projects.caliper.orchestration.export_config import ( CaliperOrchestrationExportConfig, ) @@ -203,7 +204,7 @@ def run_from_orchestration_config( else: mlflow_config_data = load_mlflow_config_yaml(Path(raw_cfg).expanduser().resolve()) - run_dirs = _discover_run_dirs(from_path) + run_dirs = discover_run_dirs(from_path) # Resolve descriptive run names from labels + run_naming config naming = resolve_run_names( @@ -217,15 +218,18 @@ def run_from_orchestration_config( ) ret = 0 else: - ret = _run_multi_run_export( - export_cfg=export_cfg, + ret = run_multi_run_artifacts_export( from_path=from_path, - status_yaml=status_yaml, + run_dirs=run_dirs, + backend=backends, + mlflow_experiment=export_cfg.mlflow_experiment, + mlflow_run_name=naming.get("parent_run_name"), mlflow_secrets_path=mlflow_secrets_path, mlflow_config_data=mlflow_config_data, - run_dirs=run_dirs, - resolved_parent_name=naming.get("parent_run_name"), child_run_names=naming.get("child_run_names") or {}, + verbose=export_cfg.verbose, + status_yaml_path=status_yaml, + upload_workers=export_cfg.upload_workers, ) else: effective_name = ( @@ -256,145 +260,3 @@ def run_from_orchestration_config( with open(status_yaml) as f: return yaml.safe_load(f.read()) - - -METRICS_FILE = "metrics.json" -PARAMETERS_FILE = "parameters.json" -TEST_LABELS_MARKER = "__test_labels__.yaml" - - -def _discover_run_dirs(from_path: Path) -> list[Path]: - """Auto-detect test run directories via ``__test_labels__.yaml`` markers.""" - run_dirs: list[Path] = [] - for marker in sorted(from_path.rglob(TEST_LABELS_MARKER)): - if marker.is_file(): - run_dirs.append(marker.parent) - - if run_dirs: - logger.info( - "Auto-detected %d test run director%s via %s", - len(run_dirs), - "y" if len(run_dirs) == 1 else "ies", - TEST_LABELS_MARKER, - ) - return run_dirs - - -def _run_multi_run_export( - *, - export_cfg: CaliperOrchestrationExportConfig, - from_path: Path, - status_yaml: Path, - mlflow_secrets_path: Path, - mlflow_config_data: dict[str, Any] | None, - run_dirs: list[Path], - resolved_parent_name: str | None = None, - child_run_names: dict[Path, str] | None = None, -) -> int: - """Export as parent + nested child MLflow runs.""" - import sys - import traceback - - import click - - from projects.caliper.engine.file_export import mlflow_backend - from projects.caliper.engine.file_export.artifacts_export_run import ( - merge_mlflow_files_with_cli, - write_artifacts_status_yaml, - ) - from projects.caliper.engine.file_export.mlflow_secrets import ( - load_mlflow_secrets_yaml, - project_secrets_fields, - validate_mlflow_secrets, - ) - from projects.caliper.engine.model import FileExportBackendResult - - logger.info("Multi-run export: %d test run(s) detected", len(run_dirs)) - - all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] - - secrets_data = None - if mlflow_secrets_path is not None: - secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) - validate_mlflow_secrets(secrets_data) - - merged_ml = merge_mlflow_files_with_cli( - None, - secrets_data=secrets_data, - config_data=mlflow_config_data, - cli_tracking_uri=None, - cli_experiment=export_cfg.mlflow_experiment, - cli_run_id=None, - cli_run_name=export_cfg.mlflow_run_name, - ) - - secret_part = project_secrets_fields(merged_ml) - mlflow_connection = secret_part if secret_part else None - - tracking_uri = merged_ml.get("tracking_uri") - experiment = merged_ml.get("experiment") - run_name = resolved_parent_name or merged_ml.get("run_name") - workspace = merged_ml.get("workspace") - if not workspace: - raise ValueError("The export workspace must be specified") - - meta = project_metadata_fields(merged_ml) - run_metadata = meta if meta else None - - insecure_tls = bool(mlflow_connection and mlflow_connection.get("insecure_tls")) - - if export_cfg.verbose: - click.echo("caliper multi-run export (verbose)", err=True) - click.echo(f" Source: {from_path}", err=True) - click.echo(f" Total artifact files: {len(all_artifact_paths)}", err=True) - click.echo(f" Run directories: {len(run_dirs)}", err=True) - click.echo(f" Workspace: {workspace}", err=True) - for rd in run_dirs: - click.echo(f" - {rd.name}", err=True) - click.echo("", err=True) - - try: - detail, ml_meta = mlflow_backend.log_multi_run_artifacts( - all_artifact_paths=all_artifact_paths, - artifact_root=from_path, - run_dirs=run_dirs, - metrics_file=METRICS_FILE, - parameters_file=PARAMETERS_FILE, - tracking_uri=tracking_uri, - experiment=experiment, - parent_run_name=run_name, - insecure_tls=insecure_tls, - connection=mlflow_connection, - verbose=export_cfg.verbose, - upload_workers=export_cfg.upload_workers, - run_metadata=run_metadata, - workspace=workspace, - child_run_names=child_run_names or None, - ) - results = [ - FileExportBackendResult( - backend="mlflow", - status="success", - detail=detail, - metadata=ml_meta, - ) - ] - except Exception as e: - traceback.print_exception(e, file=sys.stderr) - click.echo(f"multi-run export failed: {e}", err=True) - results = [FileExportBackendResult(backend="mlflow", status="failure", detail=str(e))] - - for r in results: - click.echo(f"{r.backend}: {r.status} {r.detail}") - - if status_yaml is not None: - try: - write_artifacts_status_yaml(status_yaml, results) - click.echo(f"Wrote status YAML to {status_yaml}") - except OSError as e: - click.echo(f"Failed to write status YAML ({status_yaml}): {e}", err=True) - return 4 - - if any(r.status == "failure" for r in results): - return 4 - return 0 diff --git a/projects/caliper/tests/test_multi_run_export.py b/projects/caliper/tests/test_multi_run_export.py index defa5f379..fddf67743 100644 --- a/projects/caliper/tests/test_multi_run_export.py +++ b/projects/caliper/tests/test_multi_run_export.py @@ -20,11 +20,11 @@ import pytest import yaml -from projects.caliper.orchestration.export import ( +from projects.caliper.engine.file_export.artifacts_export_run import ( METRICS_FILE, PARAMETERS_FILE, TEST_LABELS_MARKER, - _discover_run_dirs, + discover_run_dirs, ) @@ -154,13 +154,13 @@ def single_run_tree(tmp_path: Path) -> Path: # --------------------------------------------------------------------------- -# Test: _discover_run_dirs +# Test: discover_run_dirs # --------------------------------------------------------------------------- class TestDiscoverRunDirs: def test_detects_via_test_labels_markers(self, artifact_tree: Path): - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) assert len(run_dirs) == 2 names = [d.name for d in run_dirs] @@ -168,11 +168,11 @@ def test_detects_via_test_labels_markers(self, artifact_tree: Path): assert "mcp-smoke-s1-u64-gateway" in names def test_returns_empty_when_no_markers(self, single_run_tree: Path): - run_dirs = _discover_run_dirs(single_run_tree) + run_dirs = discover_run_dirs(single_run_tree) assert run_dirs == [] def test_returns_sorted(self, artifact_tree: Path): - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) names = [d.name for d in run_dirs] assert names == sorted(names) @@ -183,7 +183,7 @@ def test_ignores_dirs_without_markers(self, tmp_path: Path): (runs / "run-a").mkdir(parents=True) (runs / "run-a" / "stats.csv").write_text("data") - run_dirs = _discover_run_dirs(tmp_path / "artifacts") + run_dirs = discover_run_dirs(tmp_path / "artifacts") assert run_dirs == [] @@ -204,7 +204,7 @@ def test_all_artifacts_collected(self, artifact_tree: Path): assert PARAMETERS_FILE in all_names def test_run_dir_files_are_correct(self, artifact_tree: Path): - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) run_a = [d for d in run_dirs if "u16" in d.name][0] run_a_files = {p.name for p in run_a.rglob("*") if p.is_file()} @@ -223,7 +223,7 @@ class TestMetricsParametersLoading: def test_load_metrics_json(self, artifact_tree: Path): from projects.caliper.engine.file_export.mlflow_backend import _load_json_file - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) run_a = [d for d in run_dirs if "u16" in d.name][0] metrics = _load_json_file(run_a / METRICS_FILE) @@ -234,7 +234,7 @@ def test_load_metrics_json(self, artifact_tree: Path): def test_load_parameters_json(self, artifact_tree: Path): from projects.caliper.engine.file_export.mlflow_backend import _load_json_file - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) run_a = [d for d in run_dirs if "u16" in d.name][0] params = _load_json_file(run_a / PARAMETERS_FILE) @@ -290,7 +290,7 @@ def test_creates_parent_and_child_runs(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) all_files = [p for p in artifact_tree.rglob("*") if p.is_file()] log_multi_run_artifacts( @@ -323,7 +323,7 @@ def test_logs_metrics_from_json(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) log_multi_run_artifacts( all_artifact_paths=[], @@ -346,7 +346,7 @@ def test_logs_parameters_from_json(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) log_multi_run_artifacts( all_artifact_paths=[], @@ -372,7 +372,7 @@ def test_sets_experiment(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts( all_artifact_paths=[], artifact_root=artifact_tree, - run_dirs=_discover_run_dirs(artifact_tree), + run_dirs=discover_run_dirs(artifact_tree), metrics_file=METRICS_FILE, parameters_file=PARAMETERS_FILE, tracking_uri="http://test-mlflow:5000", @@ -387,7 +387,7 @@ def test_child_runs_only_get_own_artifacts(self, artifact_tree: Path, mock_mlflo log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) all_files = [p for p in artifact_tree.rglob("*") if p.is_file()] child_run_ids = iter(["child-run-a", "child-run-b"]) @@ -449,7 +449,7 @@ def test_sets_forge_tags_on_parent(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) log_multi_run_artifacts( all_artifact_paths=[], @@ -472,7 +472,7 @@ def test_child_run_metadata_in_status(self, artifact_tree: Path, mock_mlflow): log_multi_run_artifacts, ) - run_dirs = _discover_run_dirs(artifact_tree) + run_dirs = discover_run_dirs(artifact_tree) child_run_ids = iter(["child-id-1", "child-id-2"]) @@ -549,7 +549,7 @@ def test_workspace_set_in_env(self, artifact_tree: Path, _mock_mlflow): log_multi_run_artifacts( all_artifact_paths=[], artifact_root=artifact_tree, - run_dirs=_discover_run_dirs(artifact_tree), + run_dirs=discover_run_dirs(artifact_tree), metrics_file=METRICS_FILE, parameters_file=PARAMETERS_FILE, tracking_uri="http://test:5000", @@ -568,7 +568,7 @@ def test_no_workspace_leaves_env_unchanged(self, artifact_tree: Path, _mock_mlfl log_multi_run_artifacts( all_artifact_paths=[], artifact_root=artifact_tree, - run_dirs=_discover_run_dirs(artifact_tree), + run_dirs=discover_run_dirs(artifact_tree), metrics_file=METRICS_FILE, parameters_file=PARAMETERS_FILE, tracking_uri="http://test:5000", @@ -677,6 +677,6 @@ def test_single_marker_uses_flat_export(self, tmp_path: Path): (run_a / METRICS_FILE).write_text(json.dumps({"total_requests": 100})) (run_a / PARAMETERS_FILE).write_text(json.dumps({"preset": "smoke"})) - run_dirs = _discover_run_dirs(base) + run_dirs = discover_run_dirs(base) assert len(run_dirs) == 1 assert not (len(run_dirs) > 1) From fd389ef3d2f5432ad9e100ee97f0c889da9fbf61 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Sun, 2 Aug 2026 15:30:13 +0300 Subject: [PATCH 0363/1072] refactor: inline METRICS_FILE/PARAMETERS_FILE/TEST_LABELS_MARKER constants These were module-level constants for simple string literals that are unlikely to change. Inline them to reduce noise. Co-authored-by: Cursor --- .../engine/file_export/artifacts_export_run.py | 11 +++-------- projects/caliper/tests/test_multi_run_export.py | 11 +++++------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index 8607c9291..11e2c398a 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -254,15 +254,10 @@ def run_artifacts_export( return 0 -METRICS_FILE = "metrics.json" -PARAMETERS_FILE = "parameters.json" -TEST_LABELS_MARKER = "__test_labels__.yaml" - - def discover_run_dirs(from_path: Path) -> list[Path]: """Auto-detect test run directories via ``__test_labels__.yaml`` markers.""" run_dirs: list[Path] = [] - for marker in sorted(from_path.rglob(TEST_LABELS_MARKER)): + for marker in sorted(from_path.rglob("__test_labels__.yaml")): if marker.is_file(): run_dirs.append(marker.parent) return run_dirs @@ -373,8 +368,8 @@ def run_multi_run_artifacts_export( all_artifact_paths=all_artifact_paths, artifact_root=from_path, run_dirs=run_dirs, - metrics_file=METRICS_FILE, - parameters_file=PARAMETERS_FILE, + metrics_file="metrics.json", + parameters_file="parameters.json", tracking_uri=tracking_uri, experiment=experiment, parent_run_name=run_name, diff --git a/projects/caliper/tests/test_multi_run_export.py b/projects/caliper/tests/test_multi_run_export.py index fddf67743..f833568c5 100644 --- a/projects/caliper/tests/test_multi_run_export.py +++ b/projects/caliper/tests/test_multi_run_export.py @@ -20,12 +20,11 @@ import pytest import yaml -from projects.caliper.engine.file_export.artifacts_export_run import ( - METRICS_FILE, - PARAMETERS_FILE, - TEST_LABELS_MARKER, - discover_run_dirs, -) +from projects.caliper.engine.file_export.artifacts_export_run import discover_run_dirs + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" def _write_test_labels(directory: Path, labels: dict) -> None: From 68029534d4feb4a853219ee3872308906d36a723 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 10:44:21 +0300 Subject: [PATCH 0364/1072] factorzzation and node labeling in k8s was added as a helper function --- .../file_export/artifacts_export_run.py | 184 ++++++++++-------- projects/core/dsl/utils/k8s.py | 51 +++++ .../orchestration/preflight_phase.py | 22 ++- 3 files changed, 174 insertions(+), 83 deletions(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index 11e2c398a..bd9153712 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -95,6 +95,78 @@ def write_artifacts_status_yaml(path: Path, results: list[FileExportBackendResul ) +def _load_mlflow_inputs( + *, + mlflow_secrets_path: Path | None, + mlflow_config_data: dict[str, Any] | None, +) -> tuple[dict[str, Any] | None, dict[str, Any] | None] | int: + """Load and validate MLflow secrets + config. + + Returns ``(secrets_data, config_data)`` on success, or an int exit code on + validation failure. + """ + secrets_data: dict[str, Any] | None = None + config_data: dict[str, Any] | None = None + + if mlflow_secrets_path is not None: + try: + secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) + validate_mlflow_secrets(secrets_data) + except (OSError, ValueError, TypeError, yaml.YAMLError) as e: + click.echo(f"Invalid MLflow secrets file: {e}", err=True) + return 1 + + if mlflow_config_data is not None: + try: + config_data = dict(mlflow_config_data) + validate_mlflow_config(config_data) + except (ValueError, TypeError) as e: + click.echo(f"Invalid MLflow settings dict: {e}", err=True) + return 1 + + return secrets_data, config_data + + +def _resolve_tracking_uri(tracking_uri: str | None) -> str | int: + """Fall back to env var, validate, return URI string or int exit code.""" + if not tracking_uri: + tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") + if not tracking_uri: + click.echo( + "MLflow backend requires a tracking URI: --mlflow-endpoint / MLFLOW_TRACKING_URI, " + "or tracking_uri in --mlflow-secrets or --mlflow-config.", + err=True, + ) + return 1 + try: + assert_tracking_uri_has_no_userinfo(tracking_uri) + except ValueError as e: + click.echo(f"Invalid MLflow tracking URI: {e}", err=True) + return 1 + return tracking_uri + + +def _finalize_results( + results: list[FileExportBackendResult], + status_yaml_path: Path | None, +) -> int: + """Print results, write status YAML, return exit code.""" + for r in results: + click.echo(f"{r.backend}: {r.status} {r.detail}") + + if status_yaml_path is not None: + try: + write_artifacts_status_yaml(status_yaml_path, results) + click.echo(f"Wrote status YAML to {status_yaml_path}") + except OSError as e: + click.echo(f"Failed to write status YAML ({status_yaml_path}): {e}", err=True) + return 4 + + if any(r.status == "failure" for r in results): + return 4 + return 0 + + def run_artifacts_export( *, from_path: Path, @@ -131,24 +203,15 @@ def run_artifacts_export( ) return 1 - mlflow_connection: dict[str, Any] | None = None - secrets_data: dict[str, Any] | None = None - config_data: dict[str, Any] | None = None - if mlflow_secrets_path is not None: - try: - secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) - validate_mlflow_secrets(secrets_data) - except (OSError, ValueError, TypeError, yaml.YAMLError) as e: - click.echo(f"Invalid MLflow secrets file: {e}", err=True) - return 1 - if mlflow_config_data is not None: - try: - config_data = dict(mlflow_config_data) - validate_mlflow_config(config_data) - except (ValueError, TypeError) as e: - click.echo(f"Invalid MLflow settings dict: {e}", err=True) - return 1 + loaded = _load_mlflow_inputs( + mlflow_secrets_path=mlflow_secrets_path, + mlflow_config_data=mlflow_config_data, + ) + if isinstance(loaded, int): + return loaded + secrets_data, config_data = loaded + mlflow_connection: dict[str, Any] | None = None if secrets_data is not None or config_data is not None: merged_ml = merge_mlflow_files_with_cli( click_context, @@ -179,7 +242,6 @@ def run_artifacts_export( mlflow_run_metadata = None mlflow_workspace = None - # Ensure CLI insecure TLS flag is applied to connection if mlflow_insecure_tls: if mlflow_connection is None: mlflow_connection = {"insecure_tls": True} @@ -188,21 +250,17 @@ def run_artifacts_export( mlflow_connection["insecure_tls"] = True if "mlflow" in backends and not dry_run: - if not mlflow_tracking_uri: - mlflow_tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") - if not mlflow_tracking_uri: - click.echo( - "MLflow backend requires a tracking URI: --mlflow-endpoint / MLFLOW_TRACKING_URI, " - "or tracking_uri in --mlflow-secrets or --mlflow-config.", - err=True, - ) - return 1 - if "mlflow" in backends and mlflow_tracking_uri: + resolved = _resolve_tracking_uri(mlflow_tracking_uri) + if isinstance(resolved, int): + return resolved + mlflow_tracking_uri = resolved + elif "mlflow" in backends and mlflow_tracking_uri: try: assert_tracking_uri_has_no_userinfo(mlflow_tracking_uri) except ValueError as e: click.echo(f"Invalid MLflow tracking URI: {e}", err=True) return 1 + if verbose: click.echo("caliper artifacts export (verbose)", err=True) click.echo(f" Source: {from_path}", err=True) @@ -220,6 +278,7 @@ def run_artifacts_export( ): click.echo(line, err=True) click.echo("", err=True) + try: results = run_file_export( source=from_path, @@ -240,18 +299,8 @@ def run_artifacts_export( traceback.print_exception(e, file=sys.stderr) click.echo(f"artifacts export failed: {e}", err=True) return 4 - for r in results: - click.echo(f"{r.backend}: {r.status} {r.detail}") - if status_yaml_path is not None: - try: - write_artifacts_status_yaml(status_yaml_path, results) - click.echo(f"Wrote status YAML to {status_yaml_path}") - except OSError as e: - click.echo(f"Failed to write status YAML ({status_yaml_path}): {e}", err=True) - return 4 - if any(r.status == "failure" for r in results): - return 4 - return 0 + + return _finalize_results(results, status_yaml_path) def discover_run_dirs(from_path: Path) -> list[Path]: @@ -297,22 +346,13 @@ def run_multi_run_artifacts_export( ) return 1 - secrets_data: dict[str, Any] | None = None - config_data: dict[str, Any] | None = None - if mlflow_secrets_path is not None: - try: - secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) - validate_mlflow_secrets(secrets_data) - except (OSError, ValueError, TypeError, yaml.YAMLError) as e: - click.echo(f"Invalid MLflow secrets file: {e}", err=True) - return 1 - if mlflow_config_data is not None: - try: - config_data = dict(mlflow_config_data) - validate_mlflow_config(config_data) - except (ValueError, TypeError) as e: - click.echo(f"Invalid MLflow settings dict: {e}", err=True) - return 1 + loaded = _load_mlflow_inputs( + mlflow_secrets_path=mlflow_secrets_path, + mlflow_config_data=mlflow_config_data, + ) + if isinstance(loaded, int): + return loaded + secrets_data, config_data = loaded merged_ml = merge_mlflow_files_with_cli( None, @@ -327,7 +367,11 @@ def run_multi_run_artifacts_export( secret_part = project_secrets_fields(merged_ml) mlflow_connection = secret_part if secret_part else None - tracking_uri = merged_ml.get("tracking_uri") + resolved = _resolve_tracking_uri(merged_ml.get("tracking_uri")) + if isinstance(resolved, int): + return resolved + tracking_uri = resolved + experiment = merged_ml.get("experiment") run_name = mlflow_run_name or merged_ml.get("run_name") workspace = merged_ml.get("workspace") @@ -340,17 +384,6 @@ def run_multi_run_artifacts_export( insecure_tls = bool(mlflow_connection and mlflow_connection.get("insecure_tls")) - if not tracking_uri: - tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") - if not tracking_uri: - click.echo("MLflow backend requires a tracking URI.", err=True) - return 1 - try: - assert_tracking_uri_has_no_userinfo(tracking_uri) - except ValueError as e: - click.echo(f"Invalid MLflow tracking URI: {e}", err=True) - return 1 - all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] if verbose: @@ -394,17 +427,4 @@ def run_multi_run_artifacts_export( click.echo(f"multi-run export failed: {e}", err=True) results = [FileExportBackendResult(backend="mlflow", status="failure", detail=str(e))] - for r in results: - click.echo(f"{r.backend}: {r.status} {r.detail}") - - if status_yaml_path is not None: - try: - write_artifacts_status_yaml(status_yaml_path, results) - click.echo(f"Wrote status YAML to {status_yaml_path}") - except OSError as e: - click.echo(f"Failed to write status YAML ({status_yaml_path}): {e}", err=True) - return 4 - - if any(r.status == "failure" for r in results): - return 4 - return 0 + return _finalize_results(results, status_yaml_path) diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index a843da00c..63f02e4cc 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -378,6 +378,57 @@ def oc_cp_from_pod( return oc(*args) +def ensure_node_labels( + labels: dict[str, str], + *, + node_role: str = "worker", +) -> list[str]: + """Ensure that all nodes matching a role carry the given labels. + + For each node with the role ``node_role``, any missing labels from + ``labels`` are applied via ``oc label``. Labels that already have + the correct value are left untouched. + + Args: + labels: Mapping of label-key → label-value to enforce. + node_role: The ``node-role.kubernetes.io/`` to select nodes + (default ``"worker"``). + + Returns: + List of node names that were modified. + """ + if not labels: + return [] + + selector = f"node-role.kubernetes.io/{node_role}" + result = oc_get_json("node", selector=selector) + if not result: + logger.warning("No nodes found with role '%s'", node_role) + return [] + + nodes = result.get("items", []) + if not nodes: + logger.warning("No nodes found with role '%s'", node_role) + return [] + + modified: list[str] = [] + for node in nodes: + node_name = node["metadata"]["name"] + existing = node.get("metadata", {}).get("labels", {}) + + missing = {k: v for k, v in labels.items() if existing.get(k) != v} + if not missing: + logger.info("Node %s already has required labels", node_name) + continue + + label_args = [f"{k}={v}" for k, v in missing.items()] + logger.info("Labeling node %s with %s", node_name, ", ".join(label_args)) + oc("label", "node", node_name, *label_args, "--overwrite") + modified.append(node_name) + + return modified + + def best_effort_oc(*oc_args: str, description: str | None = None) -> None: """Run an oc command, swallowing timeout and other errors. diff --git a/projects/mcp_gateway/orchestration/preflight_phase.py b/projects/mcp_gateway/orchestration/preflight_phase.py index 0ddf8ed7f..ed7758a52 100644 --- a/projects/mcp_gateway/orchestration/preflight_phase.py +++ b/projects/mcp_gateway/orchestration/preflight_phase.py @@ -22,7 +22,7 @@ import re from projects.agentic_tools.mcp.toolbox.deploy_mock_servers import main as deploy_mock_servers -from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists +from projects.core.dsl.utils.k8s import ensure_node_labels, oc, oc_get_json, oc_resource_exists from projects.core.library import config from projects.core.orchestration.utils.k8s import ensure_namespace from projects.mcp_gateway.orchestration.runtime_config import cfg @@ -49,6 +49,7 @@ def run() -> int: ("MCP Gateway deployment running", check_gateway_deployment), ("MCPGatewayExtension reconciled", check_extension_reconciled), ("Test namespace exists", check_test_namespace), + ("Worker node scheduling labels", check_node_labels), ("Deployed version matches requested", check_version_match), ("Gateway connectivity (probe)", check_gateway_connectivity), ] @@ -189,6 +190,25 @@ def check_test_namespace() -> None: logger.info(" Namespace %s created", namespace) +def check_node_labels() -> None: + """Ensure worker nodes carry the labels required by the scheduling config. + + Reads ``infrastructure.scheduling.node_selector`` from the project config + and applies any missing labels to worker nodes so that test pods can be + scheduled. + """ + node_selector = cfg.get_scheduling_config().get("node_selector") + if not node_selector: + logger.info(" No node_selector configured, skipping") + return + + modified = ensure_node_labels(node_selector, node_role="worker") + if modified: + logger.info(" Labeled %d worker node(s): %s", len(modified), ", ".join(modified)) + else: + logger.info(" All worker nodes already have required labels") + + def check_version_match() -> None: """Verify that the deployed MCP Gateway version matches what was requested.""" requested = config.project.get_config( From 42af9a3eb5a0b1de37a7fcc37c76891084268e33 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 12:05:31 +0300 Subject: [PATCH 0365/1072] fix: sanitize secret error messages, factorize export helpers, fix duplicate /test directive - Sanitize yaml.YAMLError and secret validation error messages to prevent leaking secret file content (use e.__class__.__name__ instead of str(e)) - Extract _load_mlflow_inputs, _resolve_tracking_uri, _finalize_results helpers to eliminate ~40 lines of duplication between single and multi-run export paths - Remove /test line from mcp_gateway parsed_directives to prevent duplicate entries in GitHub notification (same pattern as /pipeline dedup in nightly) - Revert ensure_node_labels and check_node_labels (deferred to separate PR) Co-authored-by: Cursor --- .../file_export/artifacts_export_run.py | 10 +++- projects/core/dsl/utils/k8s.py | 51 ------------------- projects/mcp_gateway/orchestration/pr_args.py | 1 - .../orchestration/preflight_phase.py | 22 +------- 4 files changed, 9 insertions(+), 75 deletions(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index bd9153712..e3b9232ad 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -113,7 +113,10 @@ def _load_mlflow_inputs( secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) validate_mlflow_secrets(secrets_data) except (OSError, ValueError, TypeError, yaml.YAMLError) as e: - click.echo(f"Invalid MLflow secrets file: {e}", err=True) + click.echo( + f"Invalid MLflow secrets file: {mlflow_secrets_path} -- {e.__class__.__name__}", + err=True, + ) return 1 if mlflow_config_data is not None: @@ -227,7 +230,10 @@ def run_artifacts_export( try: validate_mlflow_secrets(secret_part) except (ValueError, TypeError) as e: - click.echo(f"Invalid MLflow configuration: {e}", err=True) + click.echo( + f"Invalid MLflow configuration -- {e.__class__.__name__}", + err=True, + ) return 1 if secrets_data is not None: mlflow_connection = secret_part if secret_part else None diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 63f02e4cc..a843da00c 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -378,57 +378,6 @@ def oc_cp_from_pod( return oc(*args) -def ensure_node_labels( - labels: dict[str, str], - *, - node_role: str = "worker", -) -> list[str]: - """Ensure that all nodes matching a role carry the given labels. - - For each node with the role ``node_role``, any missing labels from - ``labels`` are applied via ``oc label``. Labels that already have - the correct value are left untouched. - - Args: - labels: Mapping of label-key → label-value to enforce. - node_role: The ``node-role.kubernetes.io/`` to select nodes - (default ``"worker"``). - - Returns: - List of node names that were modified. - """ - if not labels: - return [] - - selector = f"node-role.kubernetes.io/{node_role}" - result = oc_get_json("node", selector=selector) - if not result: - logger.warning("No nodes found with role '%s'", node_role) - return [] - - nodes = result.get("items", []) - if not nodes: - logger.warning("No nodes found with role '%s'", node_role) - return [] - - modified: list[str] = [] - for node in nodes: - node_name = node["metadata"]["name"] - existing = node.get("metadata", {}).get("labels", {}) - - missing = {k: v for k, v in labels.items() if existing.get(k) != v} - if not missing: - logger.info("Node %s already has required labels", node_name) - continue - - label_args = [f"{k}={v}" for k, v in missing.items()] - logger.info("Labeling node %s with %s", node_name, ", ".join(label_args)) - oc("label", "node", node_name, *label_args, "--overwrite") - modified.append(node_name) - - return modified - - def best_effort_oc(*oc_args: str, description: str | None = None) -> None: """Run an oc command, swallowing timeout and other errors. diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 1523160bd..50b3f7c1f 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -58,7 +58,6 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st args = _parse_test_line(line) if args is not None: if args: - parsed_directives.append(line) logger.info("Parsed mcp_gateway presets: %s", args) continue diff --git a/projects/mcp_gateway/orchestration/preflight_phase.py b/projects/mcp_gateway/orchestration/preflight_phase.py index ed7758a52..0ddf8ed7f 100644 --- a/projects/mcp_gateway/orchestration/preflight_phase.py +++ b/projects/mcp_gateway/orchestration/preflight_phase.py @@ -22,7 +22,7 @@ import re from projects.agentic_tools.mcp.toolbox.deploy_mock_servers import main as deploy_mock_servers -from projects.core.dsl.utils.k8s import ensure_node_labels, oc, oc_get_json, oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists from projects.core.library import config from projects.core.orchestration.utils.k8s import ensure_namespace from projects.mcp_gateway.orchestration.runtime_config import cfg @@ -49,7 +49,6 @@ def run() -> int: ("MCP Gateway deployment running", check_gateway_deployment), ("MCPGatewayExtension reconciled", check_extension_reconciled), ("Test namespace exists", check_test_namespace), - ("Worker node scheduling labels", check_node_labels), ("Deployed version matches requested", check_version_match), ("Gateway connectivity (probe)", check_gateway_connectivity), ] @@ -190,25 +189,6 @@ def check_test_namespace() -> None: logger.info(" Namespace %s created", namespace) -def check_node_labels() -> None: - """Ensure worker nodes carry the labels required by the scheduling config. - - Reads ``infrastructure.scheduling.node_selector`` from the project config - and applies any missing labels to worker nodes so that test pods can be - scheduled. - """ - node_selector = cfg.get_scheduling_config().get("node_selector") - if not node_selector: - logger.info(" No node_selector configured, skipping") - return - - modified = ensure_node_labels(node_selector, node_role="worker") - if modified: - logger.info(" Labeled %d worker node(s): %s", len(modified), ", ".join(modified)) - else: - logger.info(" All worker nodes already have required labels") - - def check_version_match() -> None: """Verify that the deployed MCP Gateway version matches what was requested.""" requested = config.project.get_config( From f26b7a92d8780118bf082a603b8183c4d44c623e Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 3 Aug 2026 12:14:08 +0300 Subject: [PATCH 0366/1072] change the nightly cluster for a test --- projects/mcp_gateway/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 2c8b22a34..0623f5911 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -15,7 +15,7 @@ nightly: experiment: "mcp-gw-cpt-ghcr" version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" strip_prefix: "sha-" - cluster: "forge-smoke-testing" + cluster: "avis-cluster" version_key: "infrastructure.mcp_gateway_version" pipeline: "forge-full" preset: "demo" From 2936198d1f775df96e0b2b412cb3b255e209a7aa Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:39:09 +0200 Subject: [PATCH 0367/1072] [caliper] cli: commands: include the workspace in the artifacts export CLI --- projects/caliper/cli/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 849d2f0d7..2ccce1951 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -956,6 +956,7 @@ def kpi_s3_import( @click.option("--mlflow-experiment", default=None, envvar="MLFLOW_EXPERIMENT_NAME") @click.option("--mlflow-run-id", default=None, envvar="MLFLOW_RUN_ID") @click.option("--mlflow-run-name", default=None, envvar="CALIPER_MLFLOW_RUN_NAME") +@click.option("--mlflow-workspace", default=None, help="MLflow workspace name") @click.option("--mlflow-insecure-tls", is_flag=True) @click.option( "--mlflow-secrets", @@ -988,6 +989,7 @@ def artifacts_export( mlflow_experiment: str | None, mlflow_run_id: str | None, mlflow_run_name: str | None, + mlflow_workspace: str | None, mlflow_insecure_tls: bool, mlflow_secrets_path: Path | None, mlflow_config_path: Path | None, @@ -996,6 +998,7 @@ def artifacts_export( status_yaml_path: Path | None, upload_workers: int, ) -> None: + # Load config from file if provided, CLI args override config = {} if mlflow_config_path: @@ -1011,6 +1014,7 @@ def artifacts_export( "experiment": mlflow_experiment or config.get("experiment"), "run_id": mlflow_run_id or config.get("run_id"), "run_name": mlflow_run_name or config.get("run_name"), + "workspace": mlflow_workspace or config.get("workspace"), "insecure_tls": mlflow_insecure_tls or config.get("insecure_tls", False), "secrets_path": mlflow_secrets_path or config.get("secrets_path"), "upload_workers": upload_workers, From 5aefffed371187325a63d7e1a12a51f624938efd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:39:34 +0200 Subject: [PATCH 0368/1072] [caliper] cli: commands: fix wrong argname --- projects/caliper/cli/commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 2ccce1951..9dba9550a 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -1027,10 +1027,15 @@ def artifacts_export( dry_run=dry_run, verbose=verbose, status_yaml_path=status_yaml_path, - mlflow_config=final_config, + mlflow_config_data=final_config, ) except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() click.echo(f"artifacts export failed: {e}", err=True) + click.echo("Full traceback:", err=True) + click.echo(full_traceback, err=True) sys.exit(3) From c50c2b14ffc9b544bf4ba511b2d6c5f6b9182212 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:42:03 +0200 Subject: [PATCH 0369/1072] [caliper] engine: file_export: artifacts_export_run: wrong argname --- projects/caliper/engine/file_export/artifacts_export_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index e3b9232ad..fe38ef49e 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -136,7 +136,7 @@ def _resolve_tracking_uri(tracking_uri: str | None) -> str | int: tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") if not tracking_uri: click.echo( - "MLflow backend requires a tracking URI: --mlflow-endpoint / MLFLOW_TRACKING_URI, " + "MLflow backend requires a tracking URI: --mlflow-tracking-uri / MLFLOW_TRACKING_URI, " "or tracking_uri in --mlflow-secrets or --mlflow-config.", err=True, ) From 8697943c239b1138505bb0efd6ac87d4e0f4644d Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Tue, 4 Aug 2026 11:50:39 +0300 Subject: [PATCH 0370/1072] fix:nest indt --- .../mcp_gateway/orchestration/config.yaml | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index b28eeb441..8eda21c8b 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -72,28 +72,27 @@ caliper: enabled: true historical_kpis: historical_data output: regression_analyze/kpi_analyze.json - - s3: - bucket: psap-dashboard-data - instance: mcp_gateway - directory: nightly - vault: - name: psap-forge-aws-s3-export - aws_credentials_file: aws.credentials - import: - enabled: true - output_dir: historical_data - include_kpis_json: true - include_kpis_csv: true - include_ai_data: false - max_downloads: 50 - export: - enabled: true - upload_id: null - dry_run: false - include_csv: true - include_kpis_json: true - include_ai_data: false + s3: + bucket: psap-dashboard-data + instance: mcp_gateway + directory: nightly + vault: + name: psap-forge-aws-s3-export + aws_credentials_file: aws.credentials + import: + enabled: true + output_dir: historical_data + include_kpis_json: true + include_kpis_csv: true + include_ai_data: false + max_downloads: 50 + export: + enabled: true + upload_id: null + dry_run: false + include_csv: true + include_kpis_json: true + include_ai_data: false export: from: null From 36e9326903d30f9a4f9325463284e8124984f664 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:04:49 +0200 Subject: [PATCH 0371/1072] [agentic_tools] mcp: toolbox: deploy_mock_servers: reduce the log verbosity --- .../mcp/toolbox/deploy_mock_servers/main.py | 25 +++++++++++-------- .../toolbox/deploy_custom_catalog/main.py | 4 +-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py index 19baedb65..13693ac7e 100644 --- a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py +++ b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py @@ -12,14 +12,13 @@ from __future__ import annotations import logging -import tempfile import time from pathlib import Path from typing import Any import yaml -from projects.core.dsl import always, entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, shell, task from projects.core.dsl.utils.k8s import oc logger = logging.getLogger(__name__) @@ -45,6 +44,16 @@ def run( return 0 +@task +def setup_directories(args, ctx): + """Create command source and artifact directories""" + + shell.mkdir("src") + shell.mkdir("artifacts") + + return "Prepared src and artifacts directories" + + @task def generate_and_apply_manifests(args, ctx): """Generate YAML manifests for all servers and apply them.""" @@ -69,14 +78,10 @@ def generate_and_apply_manifests(args, ctx): combined_yaml = "---\n".join(all_manifests) src_dir = args.artifact_dir / "src" - src_dir.mkdir(parents=True, exist_ok=True) - (src_dir / "mock-servers.yaml").write_text(combined_yaml, encoding="utf-8") - - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: - tmp.write(combined_yaml) - tmp_path = tmp.name - oc("apply", "-f", tmp_path) - Path(tmp_path).unlink(missing_ok=True) + mock_servers_manifest = src_dir / "mock-servers.yaml" + mock_servers_manifest.write_text(combined_yaml, encoding="utf-8") + + oc("apply", "-f", mock_servers_manifest, log_stdout=False) return f"Applied {args.count} server manifest(s)" diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 113913a54..91c2ea738 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -40,8 +40,8 @@ def setup_directories(args, ctx): shell.mkdir("src") shell.mkdir("artifacts") - ctx.display_name = args.display_name or args.catalog_source_name - return f"Prepared directories for {ctx.display_name}" + + return "Prepared src and artifacts directories" @task From 07b7122ab3a2084b2010dff7ed0a3dd62538a949 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:06:54 +0200 Subject: [PATCH 0372/1072] [mcp_gateway] toolbox: apply_infrastructure: reduce the log verbosity --- .../toolbox/apply_infrastructure/main.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py index 062efe707..c784250de 100644 --- a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py +++ b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py @@ -11,13 +11,12 @@ from __future__ import annotations import logging -import tempfile from pathlib import Path import yaml from projects.agentic_tools.mcp.toolbox.deploy_mock_servers.main import MOCK_MCP_LABEL -from projects.core.dsl import always, entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, shell, task from projects.core.dsl.utils.k8s import oc logger = logging.getLogger(__name__) @@ -40,6 +39,16 @@ def run( return 0 +@task +def setup_directories(args, ctx): + """Create command source and artifact directories""" + + shell.mkdir("src") + shell.mkdir("artifacts") + + return "Prepared src and artifacts directories" + + @task def apply_manifests(args, ctx): """Generate and apply HTTPRoute + DestinationRule + MCPServerRegistration.""" @@ -69,19 +78,16 @@ def apply_manifests(args, ctx): combined_yaml = "---\n".join(all_manifests) src_dir = args.artifact_dir / "src" - src_dir.mkdir(parents=True, exist_ok=True) - (src_dir / "infrastructure.yaml").write_text(combined_yaml, encoding="utf-8") - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: - tmp.write(combined_yaml) - tmp_path = tmp.name - oc("apply", "-f", tmp_path) - Path(tmp_path).unlink(missing_ok=True) + infra_manifest = src_dir / "infrastructure.yaml" + infra_manifest.write_text(combined_yaml, encoding="utf-8") + + oc("apply", "-f", infra_manifest, log_stdout=False) return f"Infrastructure applied for {args.count} server(s)" -@retry(attempts=60, delay=5, backoff=1.0) +@retry(attempts=60, delay=15, backoff=1.0) @task def wait_for_registrations(args, ctx): """Wait until all N MCPServerRegistrations report Ready.""" @@ -90,7 +96,7 @@ def wait_for_registrations(args, ctx): if ready_count >= args.count: return f"All {args.count} registrations are Ready" - return (False, f"registrations ready: {ready_count}/{args.count}") + return False, f"registrations ready: {ready_count}/{args.count}" @always @@ -98,7 +104,6 @@ def wait_for_registrations(args, ctx): def capture_artifacts(args, ctx): """Capture infrastructure resource state for post-mortem diagnostics.""" artifacts_dir = args.artifact_dir / "artifacts" - artifacts_dir.mkdir(parents=True, exist_ok=True) _capture_to_file( artifacts_dir / "mcpserverregistrations.yaml", @@ -162,6 +167,7 @@ def _count_ready_registrations(*, namespace: str, api_group: str = DEFAULT_API_G "-o", 'jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\\n"}{end}', check=False, + log_stdout=False, ) if result.returncode != 0 or not result.stdout.strip(): return 0 From 7280984a4a6f92230178f230a2fe6f8f0e007f5f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:23:55 +0200 Subject: [PATCH 0373/1072] [caliper] cli: commands: save the labels and path of the test directories --- projects/caliper/cli/commands.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 9dba9550a..ee0d6535e 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -164,8 +164,15 @@ def parse_cmd( label_filter=label_filter, ) - # Extract test directories from test nodes - test_directories = [str(node.directory) for node in model.test_nodes] + # Extract test directories with labels and relative paths + test_directories = [] + for node in model.test_nodes: + test_directories.append( + { + "path": str(node.test_path), # relative to artifact_dir + "labels": node.labels, + } + ) status.update( { From 8181dec6f323ca71c5c00c1f3432f6f1bcb20758 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:24:31 +0200 Subject: [PATCH 0374/1072] [caliper] cli: commands: correctly pass the run_artifacts_export args --- projects/caliper/cli/commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index ee0d6535e..9ae472a26 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -1031,10 +1031,17 @@ def artifacts_export( run_artifacts_export( from_path=from_path, backend=list(backend) if backend else ["mlflow"], + mlflow_tracking_uri=mlflow_tracking_uri, + mlflow_experiment=mlflow_experiment, + mlflow_run_id=mlflow_run_id, + mlflow_run_name=mlflow_run_name, + mlflow_insecure_tls=mlflow_insecure_tls, + mlflow_secrets_path=mlflow_secrets_path, + mlflow_config_data=final_config, dry_run=dry_run, verbose=verbose, status_yaml_path=status_yaml_path, - mlflow_config_data=final_config, + upload_workers=upload_workers, ) except Exception as e: # noqa: BLE001 import traceback From 7c23befe79b77f7dfa439bc0378ff6460e24702c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:35:01 +0200 Subject: [PATCH 0375/1072] [caliper] Add label include/exclude filtering for all the entrypoints --- projects/caliper/cli/commands.py | 96 +++++++++++++---- projects/caliper/engine/ai_eval.py | 4 + projects/caliper/engine/label_filters.py | 41 ++++++- projects/caliper/engine/model.py | 2 +- projects/caliper/engine/parameter_matrix.py | 3 +- projects/caliper/engine/parse.py | 10 +- projects/caliper/engine/traverse.py | 83 ++++++-------- projects/caliper/engine/validation.py | 4 +- projects/caliper/engine/visualize.py | 29 +++-- projects/caliper/orchestration/cli_builder.py | 47 +++++++- .../orchestration/postprocess_config.py | 12 +++ projects/caliper/tests/stub_plugin.py | 6 +- projects/caliper/tests/test_label_filters.py | 102 ++++++++++++++++++ .../guidellm/tests/test_postprocess_parser.py | 2 +- 14 files changed, 339 insertions(+), 102 deletions(-) create mode 100644 projects/caliper/tests/test_label_filters.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 9ae472a26..3ad22c871 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -52,6 +52,21 @@ def _workspace_cli_options(cmd_func): return cmd_func +def _label_filter_options(cmd_func): + """Common include/exclude label filtering options decorator.""" + cmd_func = click.option( + "--include-label", + multiple=True, + help="Include only test directories with matching labels (format: key=value).", + )(cmd_func) + cmd_func = click.option( + "--exclude-label", + multiple=True, + help="Exclude test directories with matching labels (format: key=value).", + )(cmd_func) + return cmd_func + + def _apply_workspace_cli_overrides( ctx: click.Context, artifacts_dir: Path | None, @@ -92,9 +107,47 @@ def _plugin_tuple(ctx: click.Context): return mod, plugin +def _parse_label_filters( + include_label: tuple[str, ...], exclude_label: tuple[str, ...] +) -> tuple[list[dict[str, str]] | None, list[dict[str, str]] | None]: + """Parse include and exclude label CLI options into lists of filter dictionaries. + + Args: + include_label: Tuple of include label specs in "key=value" format + exclude_label: Tuple of exclude label specs in "key=value" format + + Returns: + Tuple of (list of include_filter_dicts, list of exclude_filter_dicts) + Each filter dict contains a single key=value pair to support multiple filters on the same key. + + Raises: + click.BadParameter: If label format is invalid + """ + from projects.caliper.engine.label_filters import parse_filter_pairs + + include_filters = None + if include_label: + try: + include_filters = parse_filter_pairs(include_label, "include") + except ValueError as e: + # Convert to click.BadParameter for CLI error handling + raise click.BadParameter(str(e)) from e + + exclude_filters = None + if exclude_label: + try: + exclude_filters = parse_filter_pairs(exclude_label, "exclude") + except ValueError as e: + # Convert to click.BadParameter for CLI error handling + raise click.BadParameter(str(e)) from e + + return include_filters, exclude_filters + + # Main commands @click.command("parse") @_workspace_cli_options +@_label_filter_options @click.option("--no-cache", is_flag=True, help="Force full parse.") @click.option( "--cache-dir", @@ -107,11 +160,6 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) -@click.option( - "--include-label", - multiple=True, - help="Include only test directories with matching labels (format: key=value).", -) @click.option( "--status-file", type=click.Path(path_type=Path), @@ -125,6 +173,7 @@ def parse_cmd( cache_dir: Path | None, show_matrix: bool, include_label: tuple[str, ...], + exclude_label: tuple[str, ...], status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, @@ -140,17 +189,8 @@ def parse_cmd( mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] - # Parse include-label options into filter dict - label_filter = None - if include_label: - label_filter = {} - for label_spec in include_label: - if "=" not in label_spec: - raise click.BadParameter( - f"Invalid label format '{label_spec}'. Use key=value format." - ) - key, value = label_spec.split("=", 1) - label_filter[key.strip()] = value.strip() + # Parse label filters + include_filter, exclude_filter = _parse_label_filters(include_label, exclude_label) status = {"success": False} @@ -161,7 +201,8 @@ def parse_cmd( plugin=plugin, use_cache=not no_cache, show_parameter_matrix=show_matrix, - label_filter=label_filter, + include_label_filter=include_filter, + exclude_label_filter=exclude_filter, ) # Extract test directories with labels and relative paths @@ -170,7 +211,7 @@ def parse_cmd( test_directories.append( { "path": str(node.test_path), # relative to artifact_dir - "labels": node.labels, + **node.test_labels, # spread test_labels directly into the object } ) @@ -228,11 +269,10 @@ def parse_cmd( @click.command("visualize") @_workspace_cli_options +@_label_filter_options @click.option("--reports", default=None, help="Comma-separated report ids.") @click.option("--report-group", default=None) @click.option("--visualize-config", type=click.Path(path_type=Path), default=None) -@click.option("--include-label", multiple=True) -@click.option("--exclude-label", multiple=True) @click.option( "--output-dir", type=click.Path(path_type=Path), @@ -388,6 +428,7 @@ def list_reports_cmd( @click.command("ai-eval-export", hidden=True) @_workspace_cli_options +@_label_filter_options @click.option("--output", type=click.Path(path_type=Path), required=True) @click.option("--no-cache", is_flag=True, help="Disable parse cache") @click.option( @@ -397,6 +438,8 @@ def list_reports_cmd( def ai_eval_export( ctx: click.Context, output: Path, + include_label: tuple[str, ...], + exclude_label: tuple[str, ...], artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -412,6 +455,9 @@ def ai_eval_export( mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + # Parse label filters + include_filter, exclude_filter = _parse_label_filters(include_label, exclude_label) + status_data = {"success": False} try: @@ -421,6 +467,8 @@ def ai_eval_export( plugin=plugin, output=output, use_cache=not no_cache, + include_label_filter=include_filter, + exclude_label_filter=exclude_filter, ) status_data = {"success": True, "output_file": str(output)} click.echo(f"Exported {output}") @@ -452,6 +500,7 @@ def ai_eval_export( # KPI commands @click.command("generate") @_workspace_cli_options +@_label_filter_options @click.option("--output", type=click.Path(path_type=Path), required=True) @click.option( "--format", @@ -468,6 +517,8 @@ def kpi_generate( ctx: click.Context, output: Path, format_type: str, + include_label: tuple[str, ...], + exclude_label: tuple[str, ...], artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -482,6 +533,9 @@ def kpi_generate( mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + # Parse label filters + include_filter, exclude_filter = _parse_label_filters(include_label, exclude_label) + status_data = {"success": False} try: @@ -491,6 +545,8 @@ def kpi_generate( plugin_module=mod, plugin=plugin, use_cache=True, + include_label_filter=include_filter, + exclude_label_filter=exclude_filter, ) # Extract test directories from test nodes diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index f22cea9ad..6b0f7d54b 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -15,6 +15,8 @@ def run_ai_eval_export( plugin: object, output: Path, use_cache: bool, + include_label_filter: list[dict[str, str]] | None = None, + exclude_label_filter: list[dict[str, str]] | None = None, ) -> dict[str, object]: """Export AI evaluation payload with structured directories and copied artifacts.""" import json # noqa: PLC0415 @@ -24,6 +26,8 @@ def run_ai_eval_export( plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, + include_label_filter=include_label_filter, + exclude_label_filter=exclude_label_filter, ) # Build base payload from plugin diff --git a/projects/caliper/engine/label_filters.py b/projects/caliper/engine/label_filters.py index 96b8c45a0..c9fb10925 100644 --- a/projects/caliper/engine/label_filters.py +++ b/projects/caliper/engine/label_filters.py @@ -17,19 +17,54 @@ def parse_filter_kv(pairs: tuple[str, ...]) -> dict[str, str]: return out +def parse_filter_pairs(pairs: tuple[str, ...], filter_type: str) -> list[dict[str, str]]: + """Parse filter pairs into individual single-key dictionaries. + + Args: + pairs: Tuple of filter pairs in "key=value" format + filter_type: Type of filter for error messages ("include" or "exclude") + + Returns: + List of single-key dictionaries, preserving repeated keys and order + + This function preserves duplicate keys and their order, unlike parse_filter_kv + which aggregates all values into a single dictionary. + """ + filters = [] + for pair in pairs: + if "=" not in pair: + raise ValueError(f"Invalid {filter_type} filter format '{pair}'. Use key=value format.") + key, value = pair.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"Invalid {filter_type} filter format '{pair}'. Use key=value format.") + filters.append({key: value}) + return filters + + def matches_filters( labels: LabelMap, *, include: dict[str, str], exclude: dict[str, str], ) -> bool: - """Exclude wins on conflict; include requires all pairs to match when non-empty.""" + """Exclude wins on conflict; include requires all pairs to match when non-empty. + + Special filter value 'not-set' matches when the field is missing from labels. + """ + + def _matches_value(labels: LabelMap, key: str, filter_value: str) -> bool: + """Check if a label matches a filter value, supporting 'not-set' for missing fields.""" + if filter_value == "not-set": + return key not in labels + return labels.get(key) == filter_value + for k, v in exclude.items(): - if labels.get(k) == v: + if _matches_value(labels, k, v): return False if not include: return True - return all(labels.get(k) == v for k, v in include.items()) + return all(_matches_value(labels, k, v) for k, v in include.items()) def filter_records( diff --git a/projects/caliper/engine/model.py b/projects/caliper/engine/model.py index 10fc42b04..c04b4c845 100644 --- a/projects/caliper/engine/model.py +++ b/projects/caliper/engine/model.py @@ -13,7 +13,7 @@ class TestBaseNode: """Directory containing __test_labels__.yaml or MatrixBenchmarking settings.yaml.""" directory: Path - labels: dict[str, Any] + test_labels: dict[str, Any] artifact_paths: list[Path] = field(default_factory=list) test_path: Path = field(default_factory=lambda: Path("")) # Path relative to base_dir diff --git a/projects/caliper/engine/parameter_matrix.py b/projects/caliper/engine/parameter_matrix.py index 03e720e10..c44f36e54 100644 --- a/projects/caliper/engine/parameter_matrix.py +++ b/projects/caliper/engine/parameter_matrix.py @@ -108,8 +108,9 @@ def format_parameter_matrix_summary(analysis: dict[str, Any], max_combinations: # Show coverage matrix lines.append("📈 Parameter Value Coverage:") for key in analysis["parameter_keys"]: - lines.append(f" {key}:") coverage = analysis["coverage_matrix"][key] + total_for_param = sum(coverage.values()) + lines.append(f" {key}: {total_for_param} records") for value, count in coverage.items(): lines.append(f" {value}: {count} records") diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index 368f712cb..e2a16db07 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -5,7 +5,6 @@ import logging import time from pathlib import Path -from typing import Any from projects.caliper.engine.cache import ( cache_path_for_test_base, @@ -34,7 +33,8 @@ def run_parse( use_cache: bool, force_report_partial: bool = True, show_parameter_matrix: bool = True, - label_filter: dict[str, Any] | None = None, + include_label_filter: list[dict[str, str]] | None = None, + exclude_label_filter: list[dict[str, str]] | None = None, ) -> UnifiedRunModel: """ Run full parse or load valid cache. @@ -44,7 +44,11 @@ def run_parse( base_dir = base_dir.resolve() # Discover test bases - nodes = discover_test_bases(base_dir, label_filter=label_filter) + nodes = discover_test_bases( + base_dir, + include_label_filter=include_label_filter, + exclude_label_filter=exclude_label_filter, + ) # Always use per-test-base caching all_records = [] diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index add739209..0f3c18a81 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -15,13 +15,17 @@ def discover_test_bases( - base_dir: Path, *, label_filter: dict[str, Any] | None = None + base_dir: Path, + *, + include_label_filter: list[dict[str, str]] | None = None, + exclude_label_filter: list[dict[str, str]] | None = None, ) -> list[TestBaseNode]: """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode. Args: base_dir: Base directory to search - label_filter: Optional dict of label key-value pairs that must match for inclusion + include_label_filter: List of filter dicts or single dict of label key-value pairs that must match for inclusion + exclude_label_filter: List of filter dicts or single dict of label key-value pairs that exclude directories if they match """ base_dir = base_dir.resolve() if not base_dir.is_dir(): @@ -42,23 +46,47 @@ def discover_test_bases( # Use hierarchical label loading for both marker types if marker_found == MARKER: - labels = _load_hierarchical_labels(path, base_dir) + test_labels = _load_hierarchical_test_labels(path, base_dir) + # For filtering, use the labels directly (hierarchical loading returns the labels dict) + # Normalize missing "labels" entry to empty mapping to allow discovery of empty marker files + labels = test_labels.get("labels", {}) else: # Use hierarchical loading for MatrixBenchmarking settings.yaml too labels = _load_hierarchical_labels_matrixbenchmarking(path, base_dir) - + test_labels = dict( + path=str(path.relative_to(base_dir)), + labels=labels, + version="matrix_benchmarking/settings", + ) # Skip directory if skip: true is set in labels if labels.get("skip") is True: continue # Apply label filtering if specified - if label_filter is not None and not _matches_label_filter(labels, label_filter): + from projects.caliper.engine.label_filters import matches_filters + + def _apply_filters(labels, include_filters, exclude_filters): + # For exclude filters, ANY match excludes (OR logic) + if exclude_filters: + for exclude_filter in exclude_filters: + if not matches_filters(labels, include={}, exclude=exclude_filter): + return False + + # For include filters, ALL must match (AND logic) + if include_filters: + for include_filter in include_filters: + if not matches_filters(labels, include=include_filter, exclude={}): + return False + + return True + + if not _apply_filters(labels, include_label_filter, exclude_label_filter): continue nodes.append( TestBaseNode( directory=path, - labels=labels, + test_labels=test_labels, artifact_paths=_list_files_under(path, exclude_markers=True), test_path=path.relative_to(base_dir), ) @@ -66,7 +94,7 @@ def discover_test_bases( return sorted(nodes, key=lambda n: str(n.directory)) -def _load_hierarchical_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]: +def _load_hierarchical_test_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]: """Load and merge __test_labels__.yaml files hierarchically from base_dir down to test_dir. Merges in order: @@ -190,47 +218,6 @@ def _load_hierarchical_labels_matrixbenchmarking(test_dir: Path, base_dir: Path) return merged_labels -def _matches_label_filter(labels: dict[str, Any], label_filter: dict[str, Any]) -> bool: - """Check if labels match the specified filter criteria. - - Args: - labels: The hierarchically merged labels from the test directory - label_filter: Dict of key-value pairs that must match - - Returns: - True if all filter criteria match, False otherwise - - Examples: - _matches_label_filter({"labels": {"platform": "CKS", "version": "3.5"}}, - {"platform": "CKS"}) -> True - _matches_label_filter({"labels": {"platform": "OCP"}}, - {"platform": "CKS"}) -> False - """ - # Extract the labels section if it exists (for __test_labels__.yaml structure) - actual_labels = labels.get("labels", {}) if isinstance(labels.get("labels"), dict) else labels - - for filter_key, filter_value in label_filter.items(): - # Support nested key access with dot notation (e.g., "labels.platform") - if "." in filter_key: - keys = filter_key.split(".") - current = labels - for key in keys: - if not isinstance(current, dict) or key not in current: - return False - current = current[key] - if current != filter_value: - return False - else: - # Direct key access in labels section or root level - if filter_key not in actual_labels and filter_key not in labels: - return False - value = actual_labels.get(filter_key, labels.get(filter_key)) - if value != filter_value: - return False - - return True - - def _deep_merge_dict(target: dict[str, Any], source: dict[str, Any]) -> None: """Deep merge source dict into target dict, with source values taking precedence.""" for key, value in source.items(): diff --git a/projects/caliper/engine/validation.py b/projects/caliper/engine/validation.py index 54a37774e..1ec181b95 100644 --- a/projects/caliper/engine/validation.py +++ b/projects/caliper/engine/validation.py @@ -34,7 +34,7 @@ def model_to_jsonable(model: UnifiedRunModel) -> dict[str, Any]: def node_to_dict(n: Any) -> dict[str, Any]: return { "directory": str(n.directory), - "labels": n.labels, + "labels": n.test_labels, "artifact_paths": [str(p) for p in n.artifact_paths], "test_path": str(n.test_path), } @@ -67,7 +67,7 @@ def model_from_jsonable(data: dict[str, Any]) -> UnifiedRunModel: nodes = [ TestBaseNode( directory=Path(n["directory"]), - labels=n["labels"], + test_labels=n["labels"], artifact_paths=[Path(p) for p in n.get("artifact_paths", [])], test_path=Path(n.get("test_path", str(Path(n["directory"]).relative_to(base_dir)))), ) diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index 529a3d03d..f4beaeb85 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -7,7 +7,7 @@ import yaml -from projects.caliper.engine.label_filters import filter_records, parse_filter_kv +from projects.caliper.engine.label_filters import parse_filter_pairs from projects.caliper.engine.parse import run_parse @@ -70,30 +70,25 @@ def run_visualize( use_cache: bool, cache_path: Path | None, ) -> list[str]: - # Convert include pairs to directory-level filter for discovery - directory_filter = None + # Parse include and exclude filters for directory-level filtering + include_filters = None + exclude_filters = None + if include_pairs: - inc = parse_filter_kv(include_pairs) - directory_filter = inc # Use include filters for directory discovery + include_filters = parse_filter_pairs(include_pairs, "include") + + if exclude_pairs: + exclude_filters = parse_filter_pairs(exclude_pairs, "exclude") - # Apply directory-level filtering during parsing + # Apply both include and exclude filtering during parsing model = run_parse( base_dir=base_dir, plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, - label_filter=directory_filter, + include_label_filter=include_filters, + exclude_label_filter=exclude_filters, ) - - # Still apply record-level filtering for exclude filters and any remaining include filters - inc = parse_filter_kv(include_pairs) - exc = parse_filter_kv(exclude_pairs) - if exc: # Only filter records if there are exclude filters - model.unified_result_records = filter_records( - model.unified_result_records, - include={}, # Include filtering already done at directory level - exclude=exc, - ) cfg = resolve_visualize_config(base_dir, visualize_config_path) ids = resolve_report_ids( reports_csv=reports_csv, diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 51cc03498..8342da7e7 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -56,6 +56,15 @@ def build_parse_command( # Enable detailed parameter matrix output cmd.append("--show-matrix") + # Include/exclude labels + if config.filtering.include_labels: + for label in config.filtering.include_labels: + cmd.extend(["--include-label", label]) + + if config.filtering.exclude_labels: + for label in config.filtering.exclude_labels: + cmd.extend(["--exclude-label", label]) + # Status file for orchestration cmd.extend(["--status-file", str(status_file)]) @@ -95,6 +104,15 @@ def build_kpi_generate_command( # Generate-specific options cmd.extend(["--output", str(output_file)]) + # Include/exclude labels (parent-level filtering) + if config.filtering.include_labels: + for label in config.filtering.include_labels: + cmd.extend(["--include-label", label]) + + if config.filtering.exclude_labels: + for label in config.filtering.exclude_labels: + cmd.extend(["--exclude-label", label]) + # Status file for orchestration cmd.extend(["--status-file", str(status_file)]) @@ -154,11 +172,25 @@ def build_visualize_command( viz_path = env.FORGE_HOME / viz_path cmd.extend(["--visualize-config", str(viz_path.resolve())]) - # Include/exclude labels - for label in config.visualize.include_labels: + # Include/exclude labels (parent-level filtering + visualize-specific) + all_include_labels = [] + all_exclude_labels = [] + + # Add parent-level filtering + if config.filtering.include_labels: + all_include_labels.extend(config.filtering.include_labels) + if config.filtering.exclude_labels: + all_exclude_labels.extend(config.filtering.exclude_labels) + + # Add visualize-specific filtering + all_include_labels.extend(config.visualize.include_labels) + all_exclude_labels.extend(config.visualize.exclude_labels) + + # Apply all filters + for label in all_include_labels: cmd.extend(["--include-label", label]) - for label in config.visualize.exclude_labels: + for label in all_exclude_labels: cmd.extend(["--exclude-label", label]) if not use_cache: @@ -252,6 +284,15 @@ def build_ai_eval_export_command( if not use_cache: cmd.append("--no-cache") + # Include/exclude labels (parent-level filtering) + if config.filtering.include_labels: + for label in config.filtering.include_labels: + cmd.extend(["--include-label", label]) + + if config.filtering.exclude_labels: + for label in config.filtering.exclude_labels: + cmd.extend(["--exclude-label", label]) + # Status file for orchestration cmd.extend(["--status-file", str(status_file)]) diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index 13f789a8a..2df47cf44 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -18,6 +18,15 @@ class CaliperOrchestrationParseSection(BaseModel): no_cache: bool = False +class CaliperOrchestrationFilteringSection(BaseModel): + """``caliper.postprocess.filtering`` — label-based test directory filtering applied across all steps.""" + + model_config = ConfigDict(extra="forbid") + + include_labels: list[str] | None = Field(default=None) + exclude_labels: list[str] | None = Field(default=None) + + class CaliperOrchestrationVisualizeSection(BaseModel): """``caliper.postprocess.visualize`` — same semantics as ``caliper visualize``.""" @@ -282,6 +291,9 @@ class CaliperOrchestrationPostprocessConfig(BaseModel): default=None, description="Explicit path to caliper.yaml manifest.", ) + filtering: CaliperOrchestrationFilteringSection = Field( + default_factory=CaliperOrchestrationFilteringSection + ) parse: CaliperOrchestrationParseSection = Field( default_factory=CaliperOrchestrationParseSection ) diff --git a/projects/caliper/tests/stub_plugin.py b/projects/caliper/tests/stub_plugin.py index adc7d10a6..899347351 100644 --- a/projects/caliper/tests/stub_plugin.py +++ b/projects/caliper/tests/stub_plugin.py @@ -20,9 +20,9 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: warnings: list[str] = [] for node in nodes: labels = ( - node.labels.get("labels") - if isinstance(node.labels.get("labels"), dict) - else node.labels + node.test_labels.get("labels") + if isinstance(node.test_labels.get("labels"), dict) + else node.test_labels ) raw = {} for p in node.artifact_paths: diff --git a/projects/caliper/tests/test_label_filters.py b/projects/caliper/tests/test_label_filters.py new file mode 100644 index 000000000..b5aa6d416 --- /dev/null +++ b/projects/caliper/tests/test_label_filters.py @@ -0,0 +1,102 @@ +"""Tests for label filtering functionality.""" + +from __future__ import annotations + +import pytest + +from projects.caliper.engine.label_filters import ( + matches_filters, + parse_filter_kv, +) + + +class TestParseFilterKv: + def test_parse_valid_pairs(self): + result = parse_filter_kv(("key1=value1", "key2=value2")) + assert result == {"key1": "value1", "key2": "value2"} + + def test_parse_with_spaces(self): + result = parse_filter_kv(("key1 = value1 ", " key2=value2")) + assert result == {"key1": "value1", "key2": "value2"} + + def test_parse_equals_in_value(self): + result = parse_filter_kv(("key1=value=with=equals",)) + assert result == {"key1": "value=with=equals"} + + def test_parse_invalid_format(self): + with pytest.raises(ValueError, match="Invalid filter"): + parse_filter_kv(("invalid_pair",)) + + +class TestMatchesFilters: + def test_include_all_match(self): + labels = {"platform": "A100", "version": "1.0"} + include = {"platform": "A100", "version": "1.0"} + exclude = {} + assert matches_filters(labels, include=include, exclude=exclude) + + def test_include_partial_match(self): + labels = {"platform": "A100", "version": "1.0"} + include = {"platform": "A100", "version": "2.0"} + exclude = {} + assert not matches_filters(labels, include=include, exclude=exclude) + + def test_exclude_match(self): + labels = {"platform": "A100", "version": "1.0"} + include = {} + exclude = {"platform": "A100"} + assert not matches_filters(labels, include=include, exclude=exclude) + + def test_exclude_wins_over_include(self): + labels = {"platform": "A100"} + include = {"platform": "A100"} + exclude = {"platform": "A100"} + assert not matches_filters(labels, include=include, exclude=exclude) + + def test_empty_include(self): + labels = {"platform": "A100"} + include = {} + exclude = {} + assert matches_filters(labels, include=include, exclude=exclude) + + def test_not_set_include_matches_missing_field(self): + """Test that 'not-set' in include filter matches when field is missing.""" + labels = {"platform": "A100"} + include = {"gpu": "not-set"} + exclude = {} + assert matches_filters(labels, include=include, exclude=exclude) + + def test_not_set_include_does_not_match_present_field(self): + """Test that 'not-set' in include filter does not match when field is present.""" + labels = {"platform": "A100", "gpu": "H100"} + include = {"gpu": "not-set"} + exclude = {} + assert not matches_filters(labels, include=include, exclude=exclude) + + def test_not_set_exclude_excludes_missing_field(self): + """Test that 'not-set' in exclude filter excludes when field is missing.""" + labels = {"platform": "A100"} + include = {} + exclude = {"gpu": "not-set"} + assert not matches_filters(labels, include=include, exclude=exclude) + + def test_not_set_exclude_does_not_exclude_present_field(self): + """Test that 'not-set' in exclude filter does not exclude when field is present.""" + labels = {"platform": "A100", "gpu": "H100"} + include = {} + exclude = {"gpu": "not-set"} + assert matches_filters(labels, include=include, exclude=exclude) + + def test_regular_value_matching_still_works(self): + """Test that regular value matching still works alongside not-set functionality.""" + labels = {"platform": "A100", "version": "1.0"} + include = {"platform": "A100", "gpu": "not-set"} + exclude = {} + assert matches_filters(labels, include=include, exclude=exclude) + + def test_not_set_with_multiple_filters(self): + """Test not-set works in combination with other filters.""" + labels = {"platform": "A100"} + include = {"platform": "A100", "gpu": "not-set"} + exclude = {"version": "not-set"} # Should exclude because version is also missing + assert not matches_filters(labels, include=include, exclude=exclude) diff --git a/projects/guidellm/tests/test_postprocess_parser.py b/projects/guidellm/tests/test_postprocess_parser.py index 65c545c38..caa357f9a 100644 --- a/projects/guidellm/tests/test_postprocess_parser.py +++ b/projects/guidellm/tests/test_postprocess_parser.py @@ -69,7 +69,7 @@ def test_parser_accepts_rate_split_benchmark_files(tmp_path: Path) -> None: parser = GuideLLMParser() node = TestBaseNode( directory=test_dir, - labels={"labels": {"guidellm_loadshape": "multi-turn"}}, + test_labels={"labels": {"guidellm_loadshape": "multi-turn"}}, artifact_paths=[file_b, file_a], ) From 5873e2fce1d2d2ad79c4d698a7a2900d864f53b1 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 08:47:03 +0200 Subject: [PATCH 0376/1072] [caliper] orchestration: always save the status file in the artifacts --- .../orchestration/caliper_invocation.py | 107 +++++++----------- projects/caliper/orchestration/postprocess.py | 18 +-- 2 files changed, 49 insertions(+), 76 deletions(-) diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py index 8acad1b07..bca01f403 100644 --- a/projects/caliper/orchestration/caliper_invocation.py +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -9,7 +9,6 @@ import logging import subprocess -import tempfile import time import traceback from pathlib import Path @@ -33,6 +32,29 @@ logger = logging.getLogger(__name__) +def _generate_automatic_status_file_path(artifacts_dir: Path, operation: str) -> Path: + """Generate automatic status file path in format: /status_files/__.yaml""" + status_dir = artifacts_dir / "status_files" + status_dir.mkdir(parents=True, exist_ok=True) + + # Find next available index + existing_files = list(status_dir.glob(f"*__{operation}.yaml")) + if existing_files: + # Extract indices from existing files + indices = [] + for f in existing_files: + try: + idx_str = f.name.split("__")[0] + indices.append(int(idx_str)) + except (ValueError, IndexError): + continue + next_idx = max(indices) + 1 if indices else 0 + else: + next_idx = 0 + + return status_dir / f"{next_idx:03d}__{operation}.yaml" + + def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. @@ -152,8 +174,8 @@ def run_artifacts_to_kpis( output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output output_file.parent.mkdir(parents=True, exist_ok=True) - # Create temporary status file for subprocess communication - status_file = output_dir / "kpi_generate_status.yaml" + # Create automatic status file path + status_file = _generate_automatic_status_file_path(output_dir, "kpi_generate") # Build CLI command command = build_kpi_generate_command( @@ -172,12 +194,6 @@ def run_artifacts_to_kpis( step_logs_dir=step_logs_dir, ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass - # Convert to expected format if result.returncode == 0 and status_data.get("success"): relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) @@ -231,8 +247,8 @@ def run_artifacts_to_ai_data( output_file = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir output_file.parent.mkdir(parents=True, exist_ok=True) - # Create temporary status file for subprocess communication - status_file = output_dir / "ai_eval_export_status.yaml" + # Create automatic status file path + status_file = _generate_automatic_status_file_path(output_dir, "ai_eval_export") # Build CLI command command = build_ai_eval_export_command( @@ -251,12 +267,6 @@ def run_artifacts_to_ai_data( step_logs_dir=step_logs_dir, ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass - # Convert to expected format if result.returncode == 0 and status_data.get("success"): relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) @@ -308,8 +318,8 @@ def run_kpis_to_csv( output_file = output_dir / postprocess_config.kpi.csv.output output_file.parent.mkdir(parents=True, exist_ok=True) - # Create temporary status file for subprocess communication - status_file = output_dir / "kpi_csv_export_status.yaml" + # Create automatic status file path + status_file = _generate_automatic_status_file_path(output_dir, "kpi_csv_export") # Build CLI command command = build_kpi_csv_export_command( @@ -329,12 +339,6 @@ def run_kpis_to_csv( step_logs_dir=step_logs_dir, ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass - # Convert to expected format if result.returncode == 0 and status_data.get("success"): relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) @@ -389,8 +393,8 @@ def run_analyse_kpis( if not historical_kpis_dir.is_absolute(): historical_kpis_dir = output_dir / historical_kpis_dir - # Create temporary status file for subprocess communication - status_file = output_dir / "analyse_kpis_status.yaml" + # Create automatic status file path + status_file = _generate_automatic_status_file_path(output_dir, "analyse_kpis") # Build CLI command command = build_analyse_kpis_command( @@ -411,11 +415,6 @@ def run_analyse_kpis( step_logs_dir=step_logs_dir, ) - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass # Convert to expected format if result.returncode == 0 and status_data.get("success"): @@ -452,9 +451,8 @@ def run_parse_step( if not config.parse.enabled: return True, {"status": "disabled", "reason": "parse disabled", "completed_at": time.time()} - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + # Create automatic status file path + status_file = _generate_automatic_status_file_path(tree_root, "parse") try: # Build CLI command @@ -503,12 +501,6 @@ def run_parse_step( "detail": f"Exception during parse: {str(e)}", "completed_at": time.time(), } - finally: - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass def run_visualize_step( @@ -530,9 +522,8 @@ def run_visualize_step( "completed_at": time.time(), } - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + # Create automatic status file path + status_file = _generate_automatic_status_file_path(tree_root, "visualize") try: # Build CLI command @@ -581,12 +572,6 @@ def run_visualize_step( "detail": f"Exception during visualize: {str(e)}", "completed_at": time.time(), } - finally: - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass def run_s3_import_step( @@ -606,9 +591,8 @@ def run_s3_import_step( "completed_at": time.time(), } - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + # Create automatic status file path + status_file = _generate_automatic_status_file_path(output_dir, "s3_import") try: # Build CLI command @@ -654,12 +638,6 @@ def run_s3_import_step( "detail": f"Exception during s3-import: {str(e)}", "completed_at": time.time(), } - finally: - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass def run_s3_export_step( @@ -682,9 +660,8 @@ def run_s3_export_step( "completed_at": time.time(), } - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + # Create automatic status file path (use step_logs_dir.parent as base artifacts dir) + status_file = _generate_automatic_status_file_path(step_logs_dir.parent, "s3_export") try: # Build CLI command @@ -733,9 +710,3 @@ def run_s3_export_step( "detail": f"Exception during s3-export: {str(e)}", "completed_at": time.time(), } - finally: - # Clean up temporary status file - try: - status_file.unlink() - except FileNotFoundError: - pass diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 9fd7b1927..3761ce3de 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -761,11 +761,12 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - import tempfile + # Create automatic status file path + from projects.caliper.orchestration.caliper_invocation import ( + _generate_automatic_status_file_path, + ) - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + status_file = _generate_automatic_status_file_path(self.tree_root, "parse") try: # Build CLI command @@ -843,11 +844,12 @@ def _run_visualize_step(self) -> None: if not self.config.visualize.enabled: return - import tempfile + # Create automatic status file path + from projects.caliper.orchestration.caliper_invocation import ( + _generate_automatic_status_file_path, + ) - # Create status file for CLI output - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: - status_file = Path(status_f.name) + status_file = _generate_automatic_status_file_path(self.tree_root, "visualize") try: # Resolve visualize output directory From 632ef41bf6d212e98489b7adf0f35fdad8d0e403 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 16:42:57 +0200 Subject: [PATCH 0377/1072] [caliper] orchestration: postprocess: fail if no test result is dectected --- projects/caliper/orchestration/postprocess.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 3761ce3de..b8297a42a 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -787,20 +787,33 @@ def _run_parse_step(self) -> None: ) if result.returncode == 0 and status_data and status_data.get("success", False): - self.parse_failed = False - self._add_step( - "parse", - { - "status": "success", - "plugin_module": status_data.get("plugin_module", "unknown"), - "record_count": status_data.get("parsed_records", 0), - "test_directories": status_data.get("test_directories", []), - "test_directories_count": status_data.get("test_directories_count", 0), - "parse_cache_ref": status_data.get("cache_ref"), - "completed_at": time.time(), - }, - log_file, - ) + record_count = status_data.get("parsed_records", 0) + + # Build common step data + step_data = { + "plugin_module": status_data.get("plugin_module", "unknown"), + "record_count": record_count, + "test_directories": status_data.get("test_directories", []), + "test_directories_count": status_data.get("test_directories_count", 0), + "parse_cache_ref": status_data.get("cache_ref"), + "completed_at": time.time(), + } + + # Check if any records were parsed - fail if none found + if record_count == 0: + self.parse_failed = True + logger.error("Caliper parse completed but found no records to process") + step_data.update( + { + "status": "failed", + "message": "No records found - parsing completed successfully but no test data was extracted", + } + ) + else: + self.parse_failed = False + step_data["status"] = "success" + + self._add_step("parse", step_data, log_file) else: self.parse_failed = True error_msg = (status_data or {}).get( From 67cd44de72fbd3c5ab564cb861142a3c4cd1c248 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 11:42:03 +0200 Subject: [PATCH 0378/1072] [caliper] engine: file_export: artifacts_export_run: wrong argname --- projects/caliper/engine/file_export/artifacts_export_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index fe38ef49e..049a69a9b 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -201,7 +201,7 @@ def run_artifacts_export( if not backends: click.echo( "Specify at least one --backend: mlflow " - "(e.g. --from ./out --backend mlflow --mlflow-endpoint http://...).", + "(e.g. --from ./out --backend mlflow --mlflow-tracking-uri http://...).", err=True, ) return 1 From caa9b421e8fceaea1bddeeccd2c2521215b8ca2f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:46:13 +0200 Subject: [PATCH 0379/1072] [guidellm] postprocess: guidellm: parsing/parsers: rename node.labels -> test_labels --- projects/guidellm/postprocess/guidellm/parsing/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index 3853ef99f..b6674e75e 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -21,7 +21,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: """Extract labels from a test node.""" - raw = node.labels + raw = node.test_labels inner = raw.get("labels") if isinstance(inner, dict): return dict(inner) @@ -32,7 +32,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: def _kpi_labels_from_node(node: TestBaseNode) -> dict[str, Any]: """Extract kpi_labels from a test node.""" - raw = node.labels + raw = node.test_labels kpi_labels = raw.get("kpi_labels") if isinstance(kpi_labels, dict): return dict(kpi_labels) From f377f63382156e0635f870394107933348a591db Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:46:34 +0200 Subject: [PATCH 0380/1072] [mcp_gateway] postprocess: mcp_gateway: parsing/parsers: rename node.labels -> test_labels --- projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py index 107f60ae5..b23c36b3f 100644 --- a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py +++ b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py @@ -23,7 +23,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: """Extract distinguishing labels from a test node.""" - raw = node.labels + raw = node.test_labels inner = raw.get("labels") if isinstance(inner, dict): return dict(inner) From de09f04f17c15164c5a962109062a22d03cba706 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:46:50 +0200 Subject: [PATCH 0381/1072] [skeleton] postprocess: default: parsing/parsers: rename node.labels -> test_labels --- projects/skeleton/postprocess/default/parsing/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/skeleton/postprocess/default/parsing/parsers.py b/projects/skeleton/postprocess/default/parsing/parsers.py index 9a2da6c4e..0331d0a77 100644 --- a/projects/skeleton/postprocess/default/parsing/parsers.py +++ b/projects/skeleton/postprocess/default/parsing/parsers.py @@ -15,7 +15,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: """Extract labels from a test node.""" - raw = node.labels + raw = node.test_labels inner = raw.get("labels") if isinstance(inner, dict): return dict(inner) From 12d4b88c0065ee1a3ad36231e0c40420954eb4fa Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 17:49:57 +0200 Subject: [PATCH 0382/1072] [llm_d] orchestration: config: add the filtering placeholders --- projects/llm_d/orchestration/config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 3ced4607f..6427287ab 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -43,6 +43,9 @@ caliper: artifacts_dir: null plugin_module: projects.guidellm.postprocess.guidellm.plugin postprocess_config: null + filtering: + include_labels: [] + exclude_labels: [] parse: enabled: true no_cache: false From 7fd0757dbc47b38a4746437d0ef197295ce81f53 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:04:04 +0200 Subject: [PATCH 0383/1072] [core] library: config: write_variables_override: # enforce list type for proper yaml serialization --- projects/core/library/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index 0d1fa1303..3216d7c3a 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -560,7 +560,7 @@ def write_variables_override(presets=None, variables_dict=None): if presets: if len(presets) == 1 and "," in presets[0]: presets = list(presets[0].split(",")) - + presets = list(presets) # enforce list type for proper yaml serialization overrides["project.args"] = presets logger.info(f"write_variables_override: project.args --> {presets}") From 3a8bd7e5fbf5ed849de89be18af90459945a48fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:05:05 +0200 Subject: [PATCH 0384/1072] [llm_d] orchestration: presets: refine the CPT presets --- .../llm_d/orchestration/presets.d/cks.yaml | 6 ++---- .../llm_d/orchestration/presets.d/cpt.yaml | 19 ++++++++++++++++--- .../llm_d/orchestration/presets.d/janus.yaml | 6 ++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml index b03451fce..b0af352e3 100644 --- a/projects/llm_d/orchestration/presets.d/cks.yaml +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -18,13 +18,11 @@ cks: gpu.nvidia.com/class: H200 cpt.kpi.labels.platform_name: CKS - cpt.kpi.labels.gpu_type: H200 - cpt.kpi.labels.test_harness: xks cks-intelligent: extends: - cks - - cpt + - cpt-xks - llama3-70b runtime.deployment_profile: @@ -38,7 +36,7 @@ cks-intelligent: cks-pd: extends: - cks - - cpt + - cpt-xks - gpt-oss runtime.deployment_profile: diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index c8924f89d..daa35f8ef 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -11,11 +11,24 @@ cpt-reference-flavors: cpt: caliper.export.backend.mlflow.config.experiment: cpt-llm-d + caliper.postprocess.s3.instance: forge-cpt + caliper.postprocess.s3.directory: llm-d/default + +cpt-xks: + extends: + - cpt + cpt.kpi.labels.test_harness: xks + caliper.postprocess.s3.directory: llm-d/xks + +cpt-rhoai-release: + extends: + - cpt cpt.kpi.labels.test_harness: rhoai-release + caliper.postprocess.s3.directory: llm-d/release cpt-release-testing-rhoai: extends: - - cpt + - cpt-rhoai-release runtime.deployment_profile: - release-distributed-default - release-precise-prefix-cache @@ -30,12 +43,12 @@ cpt-release-testing-rhoai: cpt-release-testing-gpt-oss-120b: extends: - - gpt-oss - cpt-release-testing-rhoai + - gpt-oss runtime.selected_preset: cpt-release-testing-gpt-oss-120b cpt-release-testing-llama-33-70b: extends: - - llama3-70b - cpt-release-testing-rhoai + - llama3-70b runtime.selected_preset: cpt-release-testing-llama-33-70b diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml index ed629be57..164617b77 100644 --- a/projects/llm_d/orchestration/presets.d/janus.yaml +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -11,13 +11,11 @@ janus: platform.gateway.status_address_name: gateway-internal cpt.kpi.labels.platform_name: OCP - cpt.kpi.labels.gpu_type: H200 - cpt.kpi.labels.test_harness: xks janus-intelligent: extends: - janus - - cpt + - cpt-xks - llama3-70b runtime.deployment_profile: @@ -31,7 +29,7 @@ janus-intelligent: janus-pd: extends: - janus - - cpt + - cpt-xks - gpt-oss runtime.deployment_profile: From df27fd2a2f212e391b8fdc5a67797cc58c533687 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:16:56 +0200 Subject: [PATCH 0385/1072] [mcp_gateway] postprocess: tests: test_mcp_gateway_plugin: use the new test_labels name --- .../mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py index 78f45ab95..e27a963fc 100644 --- a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py +++ b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py @@ -41,7 +41,7 @@ def _make_test_node(base_dir: Path, name: str, stats_csv: str, labels: dict) -> ) return TestBaseNode( directory=node_dir, - labels={"version": "1", "labels": labels}, + test_labels={"version": "1", "labels": labels}, artifact_paths=artifact_paths, ) @@ -97,7 +97,7 @@ def test_parse_no_stats_csv(self, tmp_path: Path): (node_dir / "master.log").write_text("log") node = TestBaseNode( directory=node_dir, - labels={"version": "1", "labels": TEST_LABELS}, + test_labels={"version": "1", "labels": TEST_LABELS}, artifact_paths=[node_dir / "master.log"], ) parser = MCPGatewayParser() From c4ef4a568d5f9fcf64577162b32051db6c44d857 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:42:08 +0200 Subject: [PATCH 0386/1072] [llm_d] orchestration: pr_args: remove outdated comment --- projects/llm_d/orchestration/pr_args.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/llm_d/orchestration/pr_args.py b/projects/llm_d/orchestration/pr_args.py index 7c11fd048..74e3fca7f 100644 --- a/projects/llm_d/orchestration/pr_args.py +++ b/projects/llm_d/orchestration/pr_args.py @@ -3,10 +3,6 @@ llm_d Project PR Arguments Parser Parses llm_d-specific directives from PR trigger comments. - -Supported syntax:: - - /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 """ from __future__ import annotations From 84e359ca1977fa0da835297a559a918a25e983ff Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:42:38 +0200 Subject: [PATCH 0387/1072] [llm_d] orchestration: presets: cpt: remove unrelated preset flag --- projects/llm_d/orchestration/presets.d/cpt.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index daa35f8ef..d8452bdfb 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -39,7 +39,6 @@ cpt-release-testing-rhoai: - heavy-heterogeneous - multi-turn model_cache.pvc.size: 300Gi - platform.operators.rhods-operator.channel: beta cpt-release-testing-gpt-oss-120b: extends: From 98d417d51fc8f075dcf3b6617cbe2674e9118430 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 09:45:07 +0200 Subject: [PATCH 0388/1072] [llm_d] orchestration: presets: models: remove the deployment_profile update --- projects/llm_d/orchestration/presets.d/models.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/orchestration/presets.d/models.yaml b/projects/llm_d/orchestration/presets.d/models.yaml index 995860310..ceac2481c 100644 --- a/projects/llm_d/orchestration/presets.d/models.yaml +++ b/projects/llm_d/orchestration/presets.d/models.yaml @@ -2,7 +2,6 @@ __multiple: true gpt-oss: runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: distributed-default runtime.smoke_request_key: default model_cache.pvc.size: 300Gi workloads.args: From ed5db17071618dcbf94f1e6eba8ab4a2e8b641e5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 10:39:45 +0200 Subject: [PATCH 0389/1072] [caliper] cli: kpi_analysis: rename analyze_kpis_with_format_conversion --- projects/caliper/cli/kpi_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/caliper/cli/kpi_analysis.py b/projects/caliper/cli/kpi_analysis.py index 4eeee8d77..a45cbf7d7 100644 --- a/projects/caliper/cli/kpi_analysis.py +++ b/projects/caliper/cli/kpi_analysis.py @@ -34,13 +34,13 @@ def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module: str) -> None: """CLI entrypoint for KPI analysis.""" from projects.caliper.engine.kpi.analyze import ( - analyze_kpis_with_format_conversion, + analyze_kpis, status_dict_to_exit_code, ) try: # Call the core analysis function from engine (returns status dict) - result = analyze_kpis_with_format_conversion( + result = analyze_kpis( current_kpis_file=current, historical_kpis_dir=historical_dir, output_file=output, From 450c8a9ed3b59d9293dda52877904a8dcabd9594 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 10:41:20 +0200 Subject: [PATCH 0390/1072] [caliper] orchestration: postprocess_config: new key 'fail_on_regression' --- projects/caliper/orchestration/postprocess_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index 2df47cf44..519d81fbe 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -136,6 +136,10 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): default="kpi_analyze.json", description="Written under post-processing artifact dir when relative.", ) + fail_on_regression: bool = Field( + default=False, + description="Whether to fail (non-zero exit code) when KPI regressions are detected (default: false).", + ) @model_validator(mode="after") def _validate_when_enabled(self) -> Self: From d248edc69919b312c2e9aaa78b6d7954a8b51b2b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 10:42:04 +0200 Subject: [PATCH 0391/1072] [projects] orchestration: config: use correct ext for the kpi_analyze.yaml file --- projects/llm_d/orchestration/config.yaml | 2 +- projects/rhaiis/orchestration/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 6427287ab..db72bc97c 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -95,7 +95,7 @@ caliper: enabled: true current_kpis: kpis/kpis.json historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json + output: regression_analyze/kpi_analyze.yaml agentic: enabled: true diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index c158eb8eb..ace921f6b 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -52,7 +52,7 @@ caliper: analyze: enabled: false historical_kpis: historical_kpis - output: regression_analysis.json + output: regression_analysis.yaml export: from: null From 2b2859d404545498675eeeab3b6bdfdb1b732441 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 10:58:03 +0200 Subject: [PATCH 0392/1072] [caliper] orchestration: caliper_invocation: add a flag to ignore the regression failure --- .../orchestration/caliper_invocation.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py index bca01f403..622784713 100644 --- a/projects/caliper/orchestration/caliper_invocation.py +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -415,16 +415,22 @@ def run_analyse_kpis( step_logs_dir=step_logs_dir, ) + # Handle fail_on_regression logic at orchestration level + if ( + hasattr(postprocess_config.analyze, "fail_on_regression") + and not postprocess_config.analyze.fail_on_regression + and result.returncode == 3 + and status_data.get("regressions_detected") + ): + # Override exit code to 0 when fail_on_regression is False but keep regression info + logger.info( + "KPI regressions detected, but fail_on_regression=False - treating as success" + ) - # Convert to expected format - if result.returncode == 0 and status_data.get("success"): - return { - "status": "success", - "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), - "completed_at": time.time(), - "log_file": log_file, - } - else: + result.returncode = 0 + status_data["success"] = True + + if not (result.returncode == 0 and status_data.get("success")): return { "status": "failed", "error": status_data.get("error", "Unknown error"), @@ -432,6 +438,13 @@ def run_analyse_kpis( "log_file": log_file, } + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "completed_at": time.time(), + "log_file": log_file, + } + except Exception as e: logger.exception("KPI analysis failed in run_analyse_kpis") return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} From 95adb993c341fa98972d7e36bffeb14605ea184d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 11:01:06 +0200 Subject: [PATCH 0393/1072] [projects] orchestration: config: set the fail_on_regression: false flag --- projects/llm_d/orchestration/config.yaml | 1 + projects/mcp_gateway/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/config.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index db72bc97c..d71e531f7 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -96,6 +96,7 @@ caliper: current_kpis: kpis/kpis.json historical_kpis: historical_data output: regression_analyze/kpi_analyze.yaml + fail_on_regression: false agentic: enabled: true diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 8eda21c8b..d43adc1da 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -72,6 +72,7 @@ caliper: enabled: true historical_kpis: historical_data output: regression_analyze/kpi_analyze.json + fail_on_regression: false s3: bucket: psap-dashboard-data instance: mcp_gateway diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index ace921f6b..7a58811db 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -53,6 +53,7 @@ caliper: enabled: false historical_kpis: historical_kpis output: regression_analysis.yaml + fail_on_regression: false export: from: null From 027c19ee2401774c402b9b5b3797c4d1df080b09 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 11:13:41 +0200 Subject: [PATCH 0394/1072] [caliper] engine: kpi: analyze: save the analyze report in JSON --- projects/caliper/engine/kpi/analyze.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index a6430af31..ba0bfa0ae 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -9,8 +9,6 @@ from pathlib import Path from typing import Any -import yaml - logger = logging.getLogger(__name__) @@ -179,14 +177,14 @@ def sort_key(r: KpiTestResult): return sorted(results, key=sort_key) -def _build_yaml_report( +def _build_report( results: list[KpiTestResult], config: AnalysisConfig, current_source: str, baseline_sources: list[str], skipped: list[dict[str, Any]], ) -> dict[str, Any]: - """Build the final YAML-serializable report structure.""" + """Build the final report structure.""" regressions = [r for r in results if r.regression] if regressions: @@ -250,12 +248,12 @@ def run_kpi_analysis( output_file: Path, plugin_module: str, ) -> dict[str, Any]: - """Run KPI regression analysis and generate a YAML report. + """Run KPI regression analysis and generate a JSON report. Args: current_kpi_file: Path to current KPI JSON file (hierarchical schema v2) historical_data_dir: Directory containing historical KPI files (kpis.json) - output_file: Path where YAML analysis report will be written + output_file: Path where JSON analysis report will be written plugin_module: Plugin module name (for loading analysis config) Returns: @@ -375,7 +373,7 @@ def run_kpi_analysis( # Build report baseline_sources = [str(p) for p in baseline_kpi_data.keys()] - report = _build_yaml_report( + report = _build_report( results=results, config=config, current_source=str(current_kpi_file), @@ -383,10 +381,10 @@ def run_kpi_analysis( skipped=skipped, ) - # Write YAML report + # Write JSON report output_file.parent.mkdir(parents=True, exist_ok=True) with open(output_file, "w") as f: - yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + json.dump(report, f, indent=2, ensure_ascii=False) regressions = report["overall"]["regression_count"] total = report["overall"]["total_tested"] @@ -450,7 +448,7 @@ def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_modu } output_file.parent.mkdir(parents=True, exist_ok=True) with open(output_file, "w") as f: - yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + json.dump(report, f, indent=2, ensure_ascii=False) def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: From 8e5247b30af7ad575f718ab58bfb178a977c3f85 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 11:14:18 +0200 Subject: [PATCH 0395/1072] [projects] orchestration: config: use correct ext for the kpi_analyze.json file --- projects/llm_d/orchestration/config.yaml | 2 +- projects/rhaiis/orchestration/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index d71e531f7..08e073db1 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -95,7 +95,7 @@ caliper: enabled: true current_kpis: kpis/kpis.json historical_kpis: historical_data - output: regression_analyze/kpi_analyze.yaml + output: regression_analyze/kpi_analyze.json fail_on_regression: false agentic: diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 7a58811db..a5dffe14f 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -52,7 +52,7 @@ caliper: analyze: enabled: false historical_kpis: historical_kpis - output: regression_analysis.yaml + output: regression_analysis.json fail_on_regression: false export: From 1660af2e4af438a04b3251ece3424e306714c89b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 14:02:19 -0400 Subject: [PATCH 0396/1072] feat: add rampup parameter to workload profiles and wire through to guidellm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add rampup: 10 to all benchmark profiles (profile1-4) and plumb the value through build_guidellm_args() → --rampup CLI flag. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 4 ++++ projects/rhaiis/orchestration/runtime_config.py | 3 +++ projects/rhaiis/orchestration/test_phase.py | 2 ++ 3 files changed, 9 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 1cee1e4c6..833cbb5e0 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -7,20 +7,24 @@ profile1: data: "prompt_tokens=1000,output_tokens=1000" rates: [1,50,100,200,300] max_seconds: 450 + rampup: 10 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] max_seconds: 450 + rampup: 10 profile3: data: "prompt_tokens=2048,output_tokens=128" rates: [1, 50, 100, 200, 300] max_seconds: 450 + rampup: 10 profile4: data: "prompt_tokens=8000,output_tokens=1000" rates: [1,25,50,75,100] max_seconds: 450 + rampup: 10 samples: 50 diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 3f898fa81..96d563fc3 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -168,6 +168,7 @@ def build_guidellm_args( data: str, rates: list[int], max_seconds: int, + rampup: int | None = None, ) -> list[str]: guidellm_args = [] for key, value in benchmark_cfg.get("args", {}).items(): @@ -178,6 +179,8 @@ def build_guidellm_args( guidellm_args.append(f"--data={data}") guidellm_args.append(f"--rate={_format_arg_value(rates)}") guidellm_args.append(f"--max-seconds={max_seconds}") + if rampup is not None: + guidellm_args.append(f"--rampup={rampup}") return guidellm_args diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 246aa4195..5c51d60b0 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -394,6 +394,7 @@ def _run_workload_benchmark( workload = runtime_config.get_workload(workload_key) rates = workload.get("rates", [1]) max_seconds = workload.get("max_seconds", 180) + rampup = workload.get("rampup") from projects.core.library import config from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -443,6 +444,7 @@ def _run_workload_benchmark( data=workload["data"], rates=rates, max_seconds=max_seconds, + rampup=rampup, ) run_guidellm_benchmark( From 6e9dd4437a85d07a4786642ad401dc18b0db259f Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 30 Jul 2026 23:03:36 -0400 Subject: [PATCH 0397/1072] feat: pre-create MLflow run to embed run ID in dashboard CSV and Slack notifications Pre-create the MLflow run during the test step (before CSV generation) so the run_id and experiment_id are available for: - Dashboard CSV columns (mlflow_run_id, mlflow_experiment_id) - Slack notifications (MLflow run URL constructed at runtime from vault) The export step resumes the pre-created run instead of creating a new one, via the existing run_id parameter on both single-run and multi-run paths. Co-authored-by: Cursor --- .gitignore | 4 +- .../file_export/artifacts_export_run.py | 2 + .../engine/file_export/mlflow_backend.py | 5 +- projects/caliper/orchestration/export.py | 1 + .../rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- projects/rhaiis/orchestration/test_phase.py | 82 +++++++++++++++++++ projects/rhaiis/postprocess/csv_export.py | 4 + projects/rhaiis/postprocess/plugin.py | 2 + projects/rhaiis/postprocess/regression.py | 57 +++++++++++++ 9 files changed, 156 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 80ea2d510..7a476c764 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,6 @@ Thumbs.db temp/ # FORGE launcher configuration (personal settings) -projects/core/launcher/launcher_config.yaml \ No newline at end of file +projects/core/launcher/launcher_config.yaml + +fournos-job-*.yaml \ No newline at end of file diff --git a/projects/caliper/engine/file_export/artifacts_export_run.py b/projects/caliper/engine/file_export/artifacts_export_run.py index 049a69a9b..bf95beb81 100644 --- a/projects/caliper/engine/file_export/artifacts_export_run.py +++ b/projects/caliper/engine/file_export/artifacts_export_run.py @@ -324,6 +324,7 @@ def run_multi_run_artifacts_export( run_dirs: list[Path], backend: tuple[str, ...] | list[str], mlflow_experiment: str | None = None, + mlflow_run_id: str | None = None, mlflow_run_name: str | None = None, mlflow_secrets_path: Path | None = None, mlflow_config_data: dict[str, Any] | None = None, @@ -411,6 +412,7 @@ def run_multi_run_artifacts_export( parameters_file="parameters.json", tracking_uri=tracking_uri, experiment=experiment, + run_id=mlflow_run_id, parent_run_name=run_name, insecure_tls=insecure_tls, connection=mlflow_connection, diff --git a/projects/caliper/engine/file_export/mlflow_backend.py b/projects/caliper/engine/file_export/mlflow_backend.py index 2d199e34c..4173f9a08 100644 --- a/projects/caliper/engine/file_export/mlflow_backend.py +++ b/projects/caliper/engine/file_export/mlflow_backend.py @@ -491,6 +491,7 @@ def log_multi_run_artifacts( parameters_file: str, tracking_uri: str | None, experiment: str | None, + run_id: str | None = None, parent_run_name: str | None = None, insecure_tls: bool = False, connection: dict[str, Any] | None = None, @@ -552,7 +553,9 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: client = mlflow.tracking.MlflowClient() start_kw: dict[str, Any] = {} - if parent_run_name: + if run_id: + start_kw["run_id"] = run_id + elif parent_run_name: start_kw["run_name"] = parent_run_name with mlflow.start_run(**start_kw) as parent: diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 59a0f4c96..cb6859f40 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -226,6 +226,7 @@ def run_from_orchestration_config( mlflow_run_name=naming.get("parent_run_name"), mlflow_secrets_path=mlflow_secrets_path, mlflow_config_data=mlflow_config_data, + mlflow_run_id=export_cfg.mlflow_run_id, child_run_names=naming.get("child_run_names") or {}, verbose=export_cfg.verbose, status_yaml_path=status_yaml, diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 89b904e48..574cbec1c 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -94,4 +94,4 @@ profiler: agent_analysis: enabled: false severity_threshold: 10 - url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" + url: "" diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 246aa4195..04d7f24b2 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -295,6 +295,18 @@ def _run_test( logger.exception("Profiler trace upload failed") _warnings.append("Profiler trace upload failed") + # Pre-create MLflow run so its URL can be embedded in test labels / CSV + mlflow_run_meta: dict[str, str] = {} + try: + mlflow_run_meta = _precreate_mlflow_run() + if mlflow_run_meta.get("run_id"): + config.project.set_config("caliper.export.mlflow_run_id", mlflow_run_meta["run_id"]) + config.project.set_config( + "caliper.export.mlflow_experiment_id", mlflow_run_meta["experiment_id"] + ) + except Exception: + logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None for wl_key in workload_keys: @@ -316,6 +328,8 @@ def _run_test( version=version, cluster_tag=cluster_tag, trtllm_config=trtllm_cfg, + mlflow_run_id=mlflow_run_meta.get("run_id", ""), + mlflow_experiment_id=mlflow_run_meta.get("experiment_id", ""), ) try: @@ -382,6 +396,8 @@ def _run_workload_benchmark( version: str, cluster_tag: str, trtllm_config: dict | None = None, + mlflow_run_id: str = "", + mlflow_experiment_id: str = "", ) -> None: """Run benchmark and post-processing for a single workload. @@ -415,6 +431,8 @@ def _run_workload_benchmark( accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, trtllm_config=trtllm_config, + mlflow_run_id=mlflow_run_id, + mlflow_experiment_id=mlflow_experiment_id, ) if not run_benchmark: @@ -471,6 +489,8 @@ def _create_test_labels( accelerator_chip: str = "", run_uuid: str = "", trtllm_config: dict | None = None, + mlflow_run_id: str = "", + mlflow_experiment_id: str = "", ) -> None: _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") parts = [f"{k}: {v}" for k, v in engine_args.items()] @@ -497,11 +517,73 @@ def _create_test_labels( "cluster_tag": cluster_tag, "runtime_args": runtime_args, "run_uuid": run_uuid, + "mlflow_run_id": mlflow_run_id, + "mlflow_experiment_id": mlflow_experiment_id, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) +def _precreate_mlflow_run() -> dict[str, str]: + """Pre-create an MLflow run so its IDs can be embedded in test labels before CSV generation. + + The run is created and immediately ended (status FINISHED). The export step + will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. + + Returns a dict with ``run_id`` and ``experiment_id`` on success, + or an empty dict if MLflow is unavailable. + """ + from projects.core.library import config + from projects.core.library import vault as vault_lib + + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False + ) + if not vault_name or not vault_key: + logger.info("MLflow vault not configured, skipping run pre-creation") + return {} + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.info("MLflow secrets file not found, skipping run pre-creation") + return {} + + experiment = config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False + ) + + import mlflow + + from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + mlflow_connection_env, + ) + + secrets_data = load_mlflow_secrets_yaml(secrets_path) + tracking_uri = secrets_data.get("tracking_uri", "") + + with mlflow_connection_env(secrets_data): + if tracking_uri: + mlflow.set_tracking_uri(tracking_uri) + if experiment: + mlflow.set_experiment(experiment) + + with mlflow.start_run(): + active = mlflow.active_run() + run_id = active.info.run_id + experiment_id = str(active.info.experiment_id) + + logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) + + return { + "run_id": run_id, + "experiment_id": experiment_id, + } + + def _set_mlflow_metadata( model_key: str, workload_key: str, diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py index 5f86396dc..4ac38a588 100644 --- a/projects/rhaiis/postprocess/csv_export.py +++ b/projects/rhaiis/postprocess/csv_export.py @@ -66,6 +66,8 @@ "prefix_tokens", "prefix_count", "request_type", + "mlflow_run_id", + "mlflow_experiment_id", ] @@ -279,4 +281,6 @@ def _total_stat(metric_name: str, stat: str): "prefix_tokens": "", "prefix_count": "", "request_type": "", + "mlflow_run_id": "", + "mlflow_experiment_id": "", } diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 3c6765053..803cbf68a 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -191,6 +191,8 @@ def export_kpis_to_csv( row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") row["guidellm_version"] = labels.get("guidellm_version", "") + row["mlflow_run_id"] = labels.get("mlflow_run_id", "") + row["mlflow_experiment_id"] = labels.get("mlflow_experiment_id", "") rows.append(row) output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index b71bec30d..0cf2e9a54 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -331,6 +331,55 @@ def run_regression_analysis( DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +def _build_mlflow_run_url() -> str: + """Construct the MLflow run URL at runtime from vault secrets and config.""" + try: + from projects.core.library import config + from projects.core.library import vault as vault_lib + + run_id = config.project.get_config( + "caliper.export.mlflow_run_id", None, print=False, warn=False + ) + experiment_id = config.project.get_config( + "caliper.export.mlflow_experiment_id", None, print=False, warn=False + ) + if not run_id or not experiment_id: + return "" + + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", + None, + print=False, + warn=False, + ) + if not vault_name or not vault_key: + return "" + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + return "" + + from projects.caliper.engine.file_export.mlflow_secrets import load_mlflow_secrets_yaml + + secrets_data = load_mlflow_secrets_yaml(secrets_path) + tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") + if not tracking_uri.startswith(("http://", "https://")): + return "" + + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + qs = f"?workspace={workspace}" if workspace else "" + return f"{tracking_uri}/#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" + except Exception: + logger.debug("Failed to build MLflow run URL", exc_info=True) + return "" + + PROFILE_DISPLAY_NAMES = { "profile1": "Profile A: Balanced (1k/1k)", "profile2": "Profile B: Variable Workload (512/2k)", @@ -492,6 +541,9 @@ def send_regression_notification( ) dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + mlflow_url = _build_mlflow_run_url() + mlflow_line = f"*MLflow:* <{mlflow_url}|View Run>\n" if mlflow_url else "" + message = ( f"{icon} *{headline}*\n" f"{user_line}" @@ -502,6 +554,7 @@ def send_regression_notification( f"*Versions:* {current_version} vs {compare_version} (baseline)\n" f"{report_line}" f"{dashboard_line}" + f"{mlflow_line}" f"*Changes:*\n{details}" ) @@ -573,6 +626,9 @@ def send_failure_notification( error_text = error if len(error) <= 500 else error[:500] + "..." + mlflow_url = _build_mlflow_run_url() + mlflow_line = f"*MLflow:* <{mlflow_url}|View Run>\n" if mlflow_url else "" + message = ( f":x: *RHAIIS Pipeline Failed*\n" f"{user_line}" @@ -583,6 +639,7 @@ def send_failure_notification( f"{version_line}" f"{cluster_line}" f"{profiles_line}" + f"{mlflow_line}" f"*Error:*\n```{error_text}```" ) From 411108070565a7ef7c52fe62bade70c27d9f9755 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 00:02:52 -0400 Subject: [PATCH 0398/1072] fix: persist pre-created MLflow run_id to disk for cross-step resume config.project.set_config() is in-memory only and doesn't survive across pipeline steps (separate process invocations). Write the pre-created run_id to a __mlflow_precreated_run__.yaml marker file that the export step discovers by scanning the artifact tree. Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 26 +++++++++++++++++++-- projects/rhaiis/orchestration/test_phase.py | 14 +++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index cb6859f40..eada0487e 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -206,6 +206,9 @@ def run_from_orchestration_config( run_dirs = discover_run_dirs(from_path) + # Resume a pre-created MLflow run if the test step left a marker + mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id(from_path) + # Resolve descriptive run names from labels + run_naming config naming = resolve_run_names( run_dirs, mlflow_config_data, fallback_run_name=export_cfg.mlflow_run_name @@ -226,7 +229,7 @@ def run_from_orchestration_config( mlflow_run_name=naming.get("parent_run_name"), mlflow_secrets_path=mlflow_secrets_path, mlflow_config_data=mlflow_config_data, - mlflow_run_id=export_cfg.mlflow_run_id, + mlflow_run_id=mlflow_run_id, child_run_names=naming.get("child_run_names") or {}, verbose=export_cfg.verbose, status_yaml_path=status_yaml, @@ -239,7 +242,7 @@ def run_from_orchestration_config( mlflow_kwargs: dict[str, Any] = { "mlflow_experiment": export_cfg.mlflow_experiment, - "mlflow_run_id": export_cfg.mlflow_run_id, + "mlflow_run_id": mlflow_run_id, "mlflow_run_name": effective_name, "mlflow_secrets_path": mlflow_secrets_path, } @@ -261,3 +264,22 @@ def run_from_orchestration_config( with open(status_yaml) as f: return yaml.safe_load(f.read()) + + +MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" + + +def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: + """Find a pre-created MLflow run_id marker written by the test step.""" + markers = sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)) + if not markers: + return None + try: + data = yaml.safe_load(markers[0].read_text(encoding="utf-8")) + run_id = data.get("run_id") if isinstance(data, dict) else None + if run_id: + logger.info("Found pre-created MLflow run_id: %s (from %s)", run_id, markers[0]) + return run_id + except (OSError, yaml.YAMLError) as e: + logger.warning("Failed to read MLflow pre-created run marker: %s", e) + return None diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 04d7f24b2..159f1e53c 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -304,6 +304,7 @@ def _run_test( config.project.set_config( "caliper.export.mlflow_experiment_id", mlflow_run_meta["experiment_id"] ) + _write_mlflow_precreated_run_marker(mlflow_run_meta) except Exception: logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) @@ -584,6 +585,19 @@ def _precreate_mlflow_run() -> dict[str, str]: } +MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" + + +def _write_mlflow_precreated_run_marker(meta: dict[str, str]) -> None: + """Persist the pre-created MLflow run_id to disk so the export step can resume it.""" + import yaml + + marker_path = env.ARTIFACT_DIR / MLFLOW_PRECREATED_RUN_MARKER + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(yaml.safe_dump(meta, sort_keys=False), encoding="utf-8") + logger.info("Wrote MLflow pre-created run marker: %s", marker_path) + + def _set_mlflow_metadata( model_key: str, workload_key: str, From 0ccd1d65473f504084fda321891cf267920757d1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 00:26:06 -0400 Subject: [PATCH 0399/1072] fix: set MLFLOW_WORKSPACE during run pre-creation The pre-created run was landing in the wrong workspace/experiment (233 instead of 264) because MLFLOW_WORKSPACE was not set. The export step sets it, so the same experiment name resolved to a different experiment ID, and the export couldn't resume the pre-created run. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 159f1e53c..1205cc833 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -555,6 +555,11 @@ def _precreate_mlflow_run() -> dict[str, str]: experiment = config.project.get_config( "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False ) + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + + import os import mlflow @@ -566,9 +571,12 @@ def _precreate_mlflow_run() -> dict[str, str]: secrets_data = load_mlflow_secrets_yaml(secrets_path) tracking_uri = secrets_data.get("tracking_uri", "") + prev_workspace = os.environ.get("MLFLOW_WORKSPACE") with mlflow_connection_env(secrets_data): if tracking_uri: mlflow.set_tracking_uri(tracking_uri) + if workspace: + os.environ["MLFLOW_WORKSPACE"] = workspace if experiment: mlflow.set_experiment(experiment) @@ -577,6 +585,11 @@ def _precreate_mlflow_run() -> dict[str, str]: run_id = active.info.run_id experiment_id = str(active.info.experiment_id) + if prev_workspace is not None: + os.environ["MLFLOW_WORKSPACE"] = prev_workspace + else: + os.environ.pop("MLFLOW_WORKSPACE", None) + logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) return { From 9d6b5a4306eb469e5a96d71c0eaeaf843bc39993 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 01:10:36 -0400 Subject: [PATCH 0400/1072] fix: read MLflow run_id from test labels as fallback for export resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marker file approach was fragile — if _write_mlflow_precreated_run_marker() failed, the exception was silently caught and the export step created a new MLflow run instead of resuming the pre-created one. Two changes: - export.py: _discover_precreated_mlflow_run_id() now falls back to reading mlflow_run_id from __test_labels__.yaml in already-discovered run_dirs - test_phase.py: marker write is isolated in its own try/except so failures are logged separately from the pre-creation itself Co-authored-by: Cursor --- .gitignore | 3 +- projects/caliper/orchestration/export.py | 60 +++++++++++++++------ projects/rhaiis/orchestration/test_phase.py | 16 +++--- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 7a476c764..565ad2d5b 100644 --- a/.gitignore +++ b/.gitignore @@ -164,4 +164,5 @@ temp/ # FORGE launcher configuration (personal settings) projects/core/launcher/launcher_config.yaml -fournos-job-*.yaml \ No newline at end of file +fournos-job-*.yaml +kubeconfig* \ No newline at end of file diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index eada0487e..8a5a85eb3 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -206,8 +206,10 @@ def run_from_orchestration_config( run_dirs = discover_run_dirs(from_path) - # Resume a pre-created MLflow run if the test step left a marker - mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id(from_path) + # Resume a pre-created MLflow run if the test step left a marker or test labels + mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id( + from_path, run_dirs + ) # Resolve descriptive run names from labels + run_naming config naming = resolve_run_names( @@ -269,17 +271,45 @@ def run_from_orchestration_config( MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" -def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: - """Find a pre-created MLflow run_id marker written by the test step.""" +def _discover_precreated_mlflow_run_id( + from_path: Path, run_dirs: list[Path] | None = None +) -> str | None: + """Find a pre-created MLflow run_id from the marker file or test labels. + + Strategy: + 1. Look for the dedicated ``__mlflow_precreated_run__.yaml`` marker. + 2. If not found, fall back to reading ``mlflow_run_id`` from the first + ``__test_labels__.yaml`` that contains it (same files that + ``_discover_run_dirs`` already found). + """ + # --- attempt 1: dedicated marker file --- markers = sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)) - if not markers: - return None - try: - data = yaml.safe_load(markers[0].read_text(encoding="utf-8")) - run_id = data.get("run_id") if isinstance(data, dict) else None - if run_id: - logger.info("Found pre-created MLflow run_id: %s (from %s)", run_id, markers[0]) - return run_id - except (OSError, yaml.YAMLError) as e: - logger.warning("Failed to read MLflow pre-created run marker: %s", e) - return None + for marker in markers: + try: + data = yaml.safe_load(marker.read_text(encoding="utf-8")) + run_id = data.get("run_id") if isinstance(data, dict) else None + if run_id: + logger.info("Found pre-created MLflow run_id: %s (from marker %s)", run_id, marker) + return run_id + except (OSError, yaml.YAMLError) as e: + logger.warning("Failed to read MLflow pre-created run marker %s: %s", marker, e) + + # --- attempt 2: read from test labels --- + TEST_LABELS_MARKER = "__test_labels__.yaml" + dirs_to_scan = run_dirs if run_dirs else [] + for run_dir in dirs_to_scan: + labels_path = run_dir / TEST_LABELS_MARKER + if not labels_path.is_file(): + continue + try: + labels = yaml.safe_load(labels_path.read_text(encoding="utf-8")) + run_id = labels.get("mlflow_run_id") if isinstance(labels, dict) else None + if run_id: + logger.info( + "Found pre-created MLflow run_id: %s (from test labels %s)", run_id, labels_path + ) + return run_id + except (OSError, yaml.YAMLError) as e: + logger.warning("Failed to read test labels %s: %s", labels_path, e) + + return None diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 1205cc833..18c809bec 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -299,15 +299,19 @@ def _run_test( mlflow_run_meta: dict[str, str] = {} try: mlflow_run_meta = _precreate_mlflow_run() - if mlflow_run_meta.get("run_id"): - config.project.set_config("caliper.export.mlflow_run_id", mlflow_run_meta["run_id"]) - config.project.set_config( - "caliper.export.mlflow_experiment_id", mlflow_run_meta["experiment_id"] - ) - _write_mlflow_precreated_run_marker(mlflow_run_meta) except Exception: logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + if mlflow_run_meta.get("run_id"): + config.project.set_config("caliper.export.mlflow_run_id", mlflow_run_meta["run_id"]) + config.project.set_config( + "caliper.export.mlflow_experiment_id", mlflow_run_meta["experiment_id"] + ) + try: + _write_mlflow_precreated_run_marker(mlflow_run_meta) + except Exception: + logger.warning("Failed to write MLflow marker file; continuing", exc_info=True) + # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None for wl_key in workload_keys: From a3342045acf9e486eae56470a6b255b6a988c563 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 01:23:08 -0400 Subject: [PATCH 0401/1072] fix: remove set_config calls for non-existent caliper.export.mlflow_run_id key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config.project.set_config() raises KeyError when the key doesn't exist in the config schema. This was the root cause of the marker file never being written — the KeyError was caught by the outer try/except before the marker write could execute. The set_config calls are unnecessary since the run_id is communicated via test labels and the marker file, not in-memory config. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 18c809bec..40751ae20 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -303,10 +303,6 @@ def _run_test( logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) if mlflow_run_meta.get("run_id"): - config.project.set_config("caliper.export.mlflow_run_id", mlflow_run_meta["run_id"]) - config.project.set_config( - "caliper.export.mlflow_experiment_id", mlflow_run_meta["experiment_id"] - ) try: _write_mlflow_precreated_run_marker(mlflow_run_meta) except Exception: From f2034b1fb18ac1c07b417bc1ea3cf36216d0b5ce Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Fri, 31 Jul 2026 01:44:50 -0400 Subject: [PATCH 0402/1072] fix: set MLflow run name to FJOB_NAME during pre-creation and resume Pre-created runs got MLflow's auto-generated name (e.g. rambunctious-fowl-43) because start_run() was called without run_name. The export step's resume path also skipped run_name when run_id was set (elif branch). Two fixes: - Pre-creation: pass FJOB_NAME as run_name to start_run() - Resume: pass both run_id and run_name so the name is updated on resume Co-authored-by: Cursor --- projects/caliper/engine/file_export/mlflow_backend.py | 4 ++-- projects/rhaiis/orchestration/test_phase.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/caliper/engine/file_export/mlflow_backend.py b/projects/caliper/engine/file_export/mlflow_backend.py index 4173f9a08..fb932518d 100644 --- a/projects/caliper/engine/file_export/mlflow_backend.py +++ b/projects/caliper/engine/file_export/mlflow_backend.py @@ -416,7 +416,7 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: start_kw: dict[str, Any] = {} if run_id: start_kw["run_id"] = run_id - elif run_name: + if run_name: start_kw["run_name"] = run_name meta: dict[str, Any] | None = None @@ -555,7 +555,7 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: start_kw: dict[str, Any] = {} if run_id: start_kw["run_id"] = run_id - elif parent_run_name: + if parent_run_name: start_kw["run_name"] = parent_run_name with mlflow.start_run(**start_kw) as parent: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 40751ae20..568616636 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -580,7 +580,8 @@ def _precreate_mlflow_run() -> dict[str, str]: if experiment: mlflow.set_experiment(experiment) - with mlflow.start_run(): + run_name = os.environ.get("FJOB_NAME") + with mlflow.start_run(run_name=run_name): active = mlflow.active_run() run_id = active.info.run_id experiment_id = str(active.info.experiment_id) From a31698fe0e97b38e8e79b6cda43ed5b74542fd8c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 2 Aug 2026 23:12:57 -0400 Subject: [PATCH 0403/1072] refactor: move MLflow helpers from rhaiis to caliper orchestration layer Addresses reviewer feedback: rhaiis orchestration/postprocess must not import caliper engine code directly. Moved to caliper/orchestration/export.py: - precreate_mlflow_run() (was _precreate_mlflow_run in test_phase.py) - write_mlflow_precreated_run_marker() (was _write_mlflow_precreated_run_marker) - build_mlflow_run_url() (was _build_mlflow_run_url in regression.py) rhaiis files now import from projects.caliper.orchestration.export. Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 125 ++++++++++++++++++++ projects/rhaiis/orchestration/test_phase.py | 96 ++------------- projects/rhaiis/postprocess/regression.py | 45 +------ 3 files changed, 134 insertions(+), 132 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 8a5a85eb3..2f425c461 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -268,9 +268,134 @@ def run_from_orchestration_config( return yaml.safe_load(f.read()) +def precreate_mlflow_run() -> dict[str, str]: + """Pre-create an MLflow run so its IDs can be embedded in test labels before CSV generation. + + The run is created and immediately ended (status FINISHED). The export step + will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. + + Returns a dict with ``run_id`` and ``experiment_id`` on success, + or an empty dict if MLflow is unavailable. + """ + from projects.core.library import config + + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False + ) + if not vault_name or not vault_key: + logger.info("MLflow vault not configured, skipping run pre-creation") + return {} + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.info("MLflow secrets file not found, skipping run pre-creation") + return {} + + experiment = config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False + ) + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + + import mlflow + + from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + mlflow_connection_env, + ) + + secrets_data = load_mlflow_secrets_yaml(secrets_path) + tracking_uri = secrets_data.get("tracking_uri", "") + + prev_workspace = os.environ.get("MLFLOW_WORKSPACE") + with mlflow_connection_env(secrets_data): + if tracking_uri: + mlflow.set_tracking_uri(tracking_uri) + if workspace: + os.environ["MLFLOW_WORKSPACE"] = workspace + if experiment: + mlflow.set_experiment(experiment) + + run_name = os.environ.get("FJOB_NAME") + with mlflow.start_run(run_name=run_name): + active = mlflow.active_run() + run_id = active.info.run_id + experiment_id = str(active.info.experiment_id) + + if prev_workspace is not None: + os.environ["MLFLOW_WORKSPACE"] = prev_workspace + else: + os.environ.pop("MLFLOW_WORKSPACE", None) + + logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) + + return { + "run_id": run_id, + "experiment_id": experiment_id, + } + + MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" +def write_mlflow_precreated_run_marker(meta: dict[str, str]) -> None: + """Persist the pre-created MLflow run_id to disk so the export step can resume it.""" + marker_path = env.ARTIFACT_DIR / MLFLOW_PRECREATED_RUN_MARKER + marker_path.parent.mkdir(parents=True, exist_ok=True) + marker_path.write_text(yaml.safe_dump(meta, sort_keys=False), encoding="utf-8") + logger.info("Wrote MLflow pre-created run marker: %s", marker_path) + + +def build_mlflow_run_url() -> str: + """Construct the MLflow run URL at runtime from vault secrets and config.""" + try: + from projects.caliper.engine.file_export.mlflow_secrets import load_mlflow_secrets_yaml + from projects.core.library import config + + run_id = config.project.get_config( + "caliper.export.mlflow_run_id", None, print=False, warn=False + ) + experiment_id = config.project.get_config( + "caliper.export.mlflow_experiment_id", None, print=False, warn=False + ) + if not run_id or not experiment_id: + return "" + + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", + None, + print=False, + warn=False, + ) + if not vault_name or not vault_key: + return "" + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + return "" + + secrets_data = load_mlflow_secrets_yaml(secrets_path) + tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") + if not tracking_uri.startswith(("http://", "https://")): + return "" + + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + qs = f"?workspace={workspace}" if workspace else "" + return f"{tracking_uri}/#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" + except Exception: + logger.debug("Failed to build MLflow run URL", exc_info=True) + return "" + + def _discover_precreated_mlflow_run_id( from_path: Path, run_dirs: list[Path] | None = None ) -> str | None: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 568616636..bbaba1d2c 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -296,15 +296,20 @@ def _run_test( _warnings.append("Profiler trace upload failed") # Pre-create MLflow run so its URL can be embedded in test labels / CSV + from projects.caliper.orchestration.export import ( + precreate_mlflow_run, + write_mlflow_precreated_run_marker, + ) + mlflow_run_meta: dict[str, str] = {} try: - mlflow_run_meta = _precreate_mlflow_run() + mlflow_run_meta = precreate_mlflow_run() except Exception: logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) if mlflow_run_meta.get("run_id"): try: - _write_mlflow_precreated_run_marker(mlflow_run_meta) + write_mlflow_precreated_run_marker(mlflow_run_meta) except Exception: logger.warning("Failed to write MLflow marker file; continuing", exc_info=True) @@ -525,93 +530,6 @@ def _create_test_labels( logger.info("Created test labels: %s", labels) -def _precreate_mlflow_run() -> dict[str, str]: - """Pre-create an MLflow run so its IDs can be embedded in test labels before CSV generation. - - The run is created and immediately ended (status FINISHED). The export step - will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. - - Returns a dict with ``run_id`` and ``experiment_id`` on success, - or an empty dict if MLflow is unavailable. - """ - from projects.core.library import config - from projects.core.library import vault as vault_lib - - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False - ) - if not vault_name or not vault_key: - logger.info("MLflow vault not configured, skipping run pre-creation") - return {} - - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - logger.info("MLflow secrets file not found, skipping run pre-creation") - return {} - - experiment = config.project.get_config( - "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False - ) - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) - - import os - - import mlflow - - from projects.caliper.engine.file_export.mlflow_secrets import ( - load_mlflow_secrets_yaml, - mlflow_connection_env, - ) - - secrets_data = load_mlflow_secrets_yaml(secrets_path) - tracking_uri = secrets_data.get("tracking_uri", "") - - prev_workspace = os.environ.get("MLFLOW_WORKSPACE") - with mlflow_connection_env(secrets_data): - if tracking_uri: - mlflow.set_tracking_uri(tracking_uri) - if workspace: - os.environ["MLFLOW_WORKSPACE"] = workspace - if experiment: - mlflow.set_experiment(experiment) - - run_name = os.environ.get("FJOB_NAME") - with mlflow.start_run(run_name=run_name): - active = mlflow.active_run() - run_id = active.info.run_id - experiment_id = str(active.info.experiment_id) - - if prev_workspace is not None: - os.environ["MLFLOW_WORKSPACE"] = prev_workspace - else: - os.environ.pop("MLFLOW_WORKSPACE", None) - - logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) - - return { - "run_id": run_id, - "experiment_id": experiment_id, - } - - -MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" - - -def _write_mlflow_precreated_run_marker(meta: dict[str, str]) -> None: - """Persist the pre-created MLflow run_id to disk so the export step can resume it.""" - import yaml - - marker_path = env.ARTIFACT_DIR / MLFLOW_PRECREATED_RUN_MARKER - marker_path.parent.mkdir(parents=True, exist_ok=True) - marker_path.write_text(yaml.safe_dump(meta, sort_keys=False), encoding="utf-8") - logger.info("Wrote MLflow pre-created run marker: %s", marker_path) - - def _set_mlflow_metadata( model_key: str, workload_key: str, diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 0cf2e9a54..a8b3f5946 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -334,50 +334,9 @@ def run_regression_analysis( def _build_mlflow_run_url() -> str: """Construct the MLflow run URL at runtime from vault secrets and config.""" - try: - from projects.core.library import config - from projects.core.library import vault as vault_lib - - run_id = config.project.get_config( - "caliper.export.mlflow_run_id", None, print=False, warn=False - ) - experiment_id = config.project.get_config( - "caliper.export.mlflow_experiment_id", None, print=False, warn=False - ) - if not run_id or not experiment_id: - return "" - - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", - None, - print=False, - warn=False, - ) - if not vault_name or not vault_key: - return "" - - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - return "" - - from projects.caliper.engine.file_export.mlflow_secrets import load_mlflow_secrets_yaml + from projects.caliper.orchestration.export import build_mlflow_run_url - secrets_data = load_mlflow_secrets_yaml(secrets_path) - tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") - if not tracking_uri.startswith(("http://", "https://")): - return "" - - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) - qs = f"?workspace={workspace}" if workspace else "" - return f"{tracking_uri}/#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" - except Exception: - logger.debug("Failed to build MLflow run URL", exc_info=True) - return "" + return build_mlflow_run_url() PROFILE_DISPLAY_NAMES = { From 376aff36f513a4c0294505643a385d6eb57a6706 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Sun, 2 Aug 2026 23:52:48 -0400 Subject: [PATCH 0404/1072] feat: add slack_notify_always for success notifications + move MLflow helpers to caliper - Add send_success_notification() for clean pipeline completions - Wire tests.rhaiis.slack_notify_always config to trigger it - Add dashboard link (gated on csv_dashboard.enabled) and MLflow link - Move precreate_mlflow_run, write_mlflow_precreated_run_marker, and build_mlflow_run_url from rhaiis to caliper/orchestration/export.py Co-authored-by: Cursor --- projects/rhaiis/orchestration/analysis.py | 25 +++++- projects/rhaiis/orchestration/config.yaml | 1 + projects/rhaiis/orchestration/test_phase.py | 59 ++++++++++----- projects/rhaiis/postprocess/regression.py | 84 +++++++++++++++++++++ 4 files changed, 148 insertions(+), 21 deletions(-) diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py index 724b99b5f..618d58540 100644 --- a/projects/rhaiis/orchestration/analysis.py +++ b/projects/rhaiis/orchestration/analysis.py @@ -165,6 +165,11 @@ def run_regression_check( restrict_profiles=restrict_profiles, ) + _ea = engine_args or {} + tp = _ea.get("tensor-parallel-size") or _ea.get("tp-size") or _ea.get("tp_size") or "" + dp = _ea.get("data-parallel-size") or _ea.get("dp-size") or "" + slack_user = config.project.get_config("tests.rhaiis.slack_user", "") + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: report_url = "" agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) @@ -181,20 +186,32 @@ def run_regression_check( from projects.rhaiis.postprocess.regression import send_regression_notification - _ea = engine_args or {} - tp = _ea.get("tensor-parallel-size") or _ea.get("tp-size") or _ea.get("tp_size") or "" - dp = _ea.get("data-parallel-size") or _ea.get("dp-size") or "" send_regression_notification( analysis, model=model_cfg.get("hf_model_id", ""), accelerator=accelerator, job_id=run_uuid, - slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + slack_user=slack_user, notification_vault="psap-forge-notifications", report_url=report_url, tp=str(tp), dp=str(dp), ) + elif config.project.get_config("tests.rhaiis.slack_notify_always", False): + from projects.rhaiis.postprocess.regression import send_success_notification + + send_success_notification( + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=run_uuid, + slack_user=slack_user, + notification_vault="psap-forge-notifications", + tp=str(tp), + dp=str(dp), + version=current_version, + workload_keys=config.project.get_config("tests.rhaiis.workload_keys", []), + cluster=config.project.get_config("rhaiis.cluster_tag", ""), + ) except Exception: logger.warning("Regression analysis failed; continuing", exc_info=True) finally: diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index a5dffe14f..3435082eb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -28,6 +28,7 @@ tests: version: "" compare_version: "" slack_user: "" + slack_notify_always: false caliper: postprocess: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index bbaba1d2c..4bcc1c520 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -619,27 +619,52 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) version = config.project.get_config("tests.rhaiis.version", "") compare_version = config.project.get_config("tests.rhaiis.compare_version", "") - if not compare_version or not version: + + if compare_version and version: + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) + first_workload = runtime_config.get_workload(workload_keys[0]) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) + + from projects.rhaiis.orchestration.analysis import run_regression_check + + run_regression_check( + csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid="", + engine_args=ea, + ) return - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - engine = runtime_config.get_engine() - engine_defaults = runtime_config.get_engine_args(engine) - first_workload = runtime_config.get_workload(workload_keys[0]) - ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) + if config.project.get_config("tests.rhaiis.slack_notify_always", False): + from projects.rhaiis.postprocess.regression import send_success_notification - from projects.rhaiis.orchestration.analysis import run_regression_check + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) + first_workload = runtime_config.get_workload(workload_keys[0]) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) + tp = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or "" + dp = ea.get("data-parallel-size") or ea.get("dp-size") or "" - run_regression_check( - csv_path, - compare_version, - version, - model_cfg, - accelerator, - run_uuid="", - engine_args=ea, - ) + send_success_notification( + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(tp), + dp=str(dp), + version=version, + workload_keys=workload_keys, + cluster=config.project.get_config("rhaiis.cluster_tag", ""), + ) def _upload_predictor_log(run_uuid: str) -> None: diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index a8b3f5946..565da1af1 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -526,6 +526,90 @@ def send_regression_notification( ) +def send_success_notification( + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + notification_vault: str | None = None, + dry_run: bool = False, + tp: str = "", + dp: str = "", + version: str = "", + workload_keys: list[str] | None = None, + cluster: str = "", +) -> bool: + """Send a Slack notification when the RHAIIS pipeline succeeds with no regressions. + + Returns: + True if notification sent successfully + """ + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + parallelism_line = ( + f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + ) + + profiles_line = "" + if workload_keys: + profiles_line = f"*Workloads:* {', '.join(workload_keys)}\n" + + cluster_line = f"*Cluster:* {cluster}\n" if cluster else "" + version_line = f"*Version:* {version}\n" if version else "" + + dashboard_line = "" + try: + from projects.core.library import config + + if config.project.get_config("caliper.postprocess.csv_dashboard.enabled", False): + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=version, + profiles=workload_keys, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + except Exception: + pass + + mlflow_url = _build_mlflow_run_url() + mlflow_line = f"*MLflow:* <{mlflow_url}|View Run>\n" if mlflow_url else "" + + message = ( + f":white_check_mark: *RHAIIS Pipeline Succeeded*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"{version_line}" + f"{cluster_line}" + f"{profiles_line}" + f"{dashboard_line}" + f"{mlflow_line}" + ) + + if dry_run: + logger.info("DRY RUN success notification:\n%s", message) + return True + + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) + + def send_failure_notification( *, error: str, From 89e7854ec18b7102770d60af78db4ae5eb0d509c Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 00:08:18 -0400 Subject: [PATCH 0405/1072] fix: read MLflow IDs from marker file in build_mlflow_run_url() The config keys caliper.export.mlflow_run_id/experiment_id don't exist in the schema, so get_config returns None and the URL is never built. Now reads from the __mlflow_precreated_run__.yaml marker file on disk. Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 2f425c461..184f5f9a8 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -350,18 +350,28 @@ def write_mlflow_precreated_run_marker(meta: dict[str, str]) -> None: logger.info("Wrote MLflow pre-created run marker: %s", marker_path) +def _read_mlflow_ids_from_marker() -> tuple[str, str]: + """Read run_id and experiment_id from the pre-created marker file on disk.""" + try: + artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None + if not artifact_dir: + return "", "" + for marker in sorted(artifact_dir.rglob(MLFLOW_PRECREATED_RUN_MARKER)): + data = yaml.safe_load(marker.read_text(encoding="utf-8")) + if isinstance(data, dict) and data.get("run_id"): + return data["run_id"], data.get("experiment_id", "") + except Exception: + pass + return "", "" + + def build_mlflow_run_url() -> str: - """Construct the MLflow run URL at runtime from vault secrets and config.""" + """Construct the MLflow run URL at runtime from vault secrets and the marker file.""" try: from projects.caliper.engine.file_export.mlflow_secrets import load_mlflow_secrets_yaml from projects.core.library import config - run_id = config.project.get_config( - "caliper.export.mlflow_run_id", None, print=False, warn=False - ) - experiment_id = config.project.get_config( - "caliper.export.mlflow_experiment_id", None, print=False, warn=False - ) + run_id, experiment_id = _read_mlflow_ids_from_marker() if not run_id or not experiment_id: return "" From b61992be1f85facbbccb9adb345f66a4a71001f8 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 20:09:02 -0400 Subject: [PATCH 0406/1072] fix: address CodeRabbit review findings - mlflow_backend.py: set mlflow.runName tag explicitly when resuming a run with run_id, since some MLflow versions ignore run_name in start_run() during resume - export.py: wrap MLFLOW_WORKSPACE restoration in finally block so it executes even if set_experiment/start_run raises - export.py: validate tracking_uri with assert_tracking_uri_has_no_userinfo() before composing the Slack-visible MLflow URL Co-authored-by: Cursor --- .../engine/file_export/mlflow_backend.py | 4 ++ projects/caliper/orchestration/export.py | 42 ++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/projects/caliper/engine/file_export/mlflow_backend.py b/projects/caliper/engine/file_export/mlflow_backend.py index fb932518d..b88b9be27 100644 --- a/projects/caliper/engine/file_export/mlflow_backend.py +++ b/projects/caliper/engine/file_export/mlflow_backend.py @@ -423,6 +423,8 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: client = mlflow.tracking.MlflowClient() with mlflow.start_run(**start_kw): rid = mlflow.active_run().info.run_id + if run_id and run_name: + mlflow.set_tag("mlflow.runName", run_name) _apply_run_metadata(effective_meta) _apply_log_model(artifact_root, effective_meta, verbose=verbose) @@ -560,6 +562,8 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: with mlflow.start_run(**start_kw) as parent: parent_rid = parent.info.run_id + if run_id and parent_run_name: + mlflow.set_tag("mlflow.runName", parent_run_name) _apply_run_metadata(effective_meta) _upload_mlflow_files_parallel( diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 184f5f9a8..43b90af60 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -312,24 +312,25 @@ def precreate_mlflow_run() -> dict[str, str]: tracking_uri = secrets_data.get("tracking_uri", "") prev_workspace = os.environ.get("MLFLOW_WORKSPACE") - with mlflow_connection_env(secrets_data): - if tracking_uri: - mlflow.set_tracking_uri(tracking_uri) - if workspace: - os.environ["MLFLOW_WORKSPACE"] = workspace - if experiment: - mlflow.set_experiment(experiment) - - run_name = os.environ.get("FJOB_NAME") - with mlflow.start_run(run_name=run_name): - active = mlflow.active_run() - run_id = active.info.run_id - experiment_id = str(active.info.experiment_id) - - if prev_workspace is not None: - os.environ["MLFLOW_WORKSPACE"] = prev_workspace - else: - os.environ.pop("MLFLOW_WORKSPACE", None) + try: + with mlflow_connection_env(secrets_data): + if tracking_uri: + mlflow.set_tracking_uri(tracking_uri) + if workspace: + os.environ["MLFLOW_WORKSPACE"] = workspace + if experiment: + mlflow.set_experiment(experiment) + + run_name = os.environ.get("FJOB_NAME") + with mlflow.start_run(run_name=run_name): + active = mlflow.active_run() + run_id = active.info.run_id + experiment_id = str(active.info.experiment_id) + finally: + if prev_workspace is not None: + os.environ["MLFLOW_WORKSPACE"] = prev_workspace + else: + os.environ.pop("MLFLOW_WORKSPACE", None) logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) @@ -368,7 +369,9 @@ def _read_mlflow_ids_from_marker() -> tuple[str, str]: def build_mlflow_run_url() -> str: """Construct the MLflow run URL at runtime from vault secrets and the marker file.""" try: - from projects.caliper.engine.file_export.mlflow_secrets import load_mlflow_secrets_yaml + from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + ) from projects.core.library import config run_id, experiment_id = _read_mlflow_ids_from_marker() @@ -395,6 +398,7 @@ def build_mlflow_run_url() -> str: tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") if not tracking_uri.startswith(("http://", "https://")): return "" + assert_tracking_uri_has_no_userinfo(tracking_uri) workspace = config.project.get_config( "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False From 85247981c58e62da062600aaf37af919704c7b0e Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 20:42:45 -0400 Subject: [PATCH 0407/1072] simplify: drop test-labels fallback from _discover_precreated_mlflow_run_id The marker file is always written reliably; the test-labels fallback was redundant defensive code. Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 39 +++--------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 43b90af60..a43b5c6f3 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -207,9 +207,7 @@ def run_from_orchestration_config( run_dirs = discover_run_dirs(from_path) # Resume a pre-created MLflow run if the test step left a marker or test labels - mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id( - from_path, run_dirs - ) + mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id(from_path) # Resolve descriptive run names from labels + run_naming config naming = resolve_run_names( @@ -410,20 +408,9 @@ def build_mlflow_run_url() -> str: return "" -def _discover_precreated_mlflow_run_id( - from_path: Path, run_dirs: list[Path] | None = None -) -> str | None: - """Find a pre-created MLflow run_id from the marker file or test labels. - - Strategy: - 1. Look for the dedicated ``__mlflow_precreated_run__.yaml`` marker. - 2. If not found, fall back to reading ``mlflow_run_id`` from the first - ``__test_labels__.yaml`` that contains it (same files that - ``_discover_run_dirs`` already found). - """ - # --- attempt 1: dedicated marker file --- - markers = sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)) - for marker in markers: +def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: + """Find a pre-created MLflow run_id from the marker file.""" + for marker in sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)): try: data = yaml.safe_load(marker.read_text(encoding="utf-8")) run_id = data.get("run_id") if isinstance(data, dict) else None @@ -433,22 +420,4 @@ def _discover_precreated_mlflow_run_id( except (OSError, yaml.YAMLError) as e: logger.warning("Failed to read MLflow pre-created run marker %s: %s", marker, e) - # --- attempt 2: read from test labels --- - TEST_LABELS_MARKER = "__test_labels__.yaml" - dirs_to_scan = run_dirs if run_dirs else [] - for run_dir in dirs_to_scan: - labels_path = run_dir / TEST_LABELS_MARKER - if not labels_path.is_file(): - continue - try: - labels = yaml.safe_load(labels_path.read_text(encoding="utf-8")) - run_id = labels.get("mlflow_run_id") if isinstance(labels, dict) else None - if run_id: - logger.info( - "Found pre-created MLflow run_id: %s (from test labels %s)", run_id, labels_path - ) - return run_id - except (OSError, yaml.YAMLError) as e: - logger.warning("Failed to read test labels %s: %s", labels_path, e) - return None From cd6cad7865d41bb1cc20cfd7b42441e31347929e Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 20:45:47 -0400 Subject: [PATCH 0408/1072] fix: add missing import for assert_tracking_uri_has_no_userinfo Import was lost during rebase when build_mlflow_run_url() was rewritten to read from the marker file. Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index a43b5c6f3..d6652993e 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -368,6 +368,7 @@ def build_mlflow_run_url() -> str: """Construct the MLflow run URL at runtime from vault secrets and the marker file.""" try: from projects.caliper.engine.file_export.mlflow_secrets import ( + assert_tracking_uri_has_no_userinfo, load_mlflow_secrets_yaml, ) from projects.core.library import config From 09ec49aee31e8faafcc43a77ff0280ce79612431 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 23:36:11 -0400 Subject: [PATCH 0409/1072] feat: add rampup parameter to all workload profiles Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/workloads.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 1cee1e4c6..833cbb5e0 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -7,20 +7,24 @@ profile1: data: "prompt_tokens=1000,output_tokens=1000" rates: [1,50,100,200,300] max_seconds: 450 + rampup: 10 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] max_seconds: 450 + rampup: 10 profile3: data: "prompt_tokens=2048,output_tokens=128" rates: [1, 50, 100, 200, 300] max_seconds: 450 + rampup: 10 profile4: data: "prompt_tokens=8000,output_tokens=1000" rates: [1,25,50,75,100] max_seconds: 450 + rampup: 10 samples: 50 From 48aa32f41ec728b68d2501e65bc195ee83ee2497 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 3 Aug 2026 23:51:33 -0400 Subject: [PATCH 0410/1072] fix: wire rampup workload field through to guidellm CLI args Co-authored-by: Cursor --- projects/rhaiis/orchestration/runtime_config.py | 3 +++ projects/rhaiis/orchestration/test_phase.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 3f898fa81..96d563fc3 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -168,6 +168,7 @@ def build_guidellm_args( data: str, rates: list[int], max_seconds: int, + rampup: int | None = None, ) -> list[str]: guidellm_args = [] for key, value in benchmark_cfg.get("args", {}).items(): @@ -178,6 +179,8 @@ def build_guidellm_args( guidellm_args.append(f"--data={data}") guidellm_args.append(f"--rate={_format_arg_value(rates)}") guidellm_args.append(f"--max-seconds={max_seconds}") + if rampup is not None: + guidellm_args.append(f"--rampup={rampup}") return guidellm_args diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 4bcc1c520..185b63931 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -416,6 +416,7 @@ def _run_workload_benchmark( workload = runtime_config.get_workload(workload_key) rates = workload.get("rates", [1]) max_seconds = workload.get("max_seconds", 180) + rampup = workload.get("rampup") from projects.core.library import config from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -467,6 +468,7 @@ def _run_workload_benchmark( data=workload["data"], rates=rates, max_seconds=max_seconds, + rampup=rampup, ) run_guidellm_benchmark( From 951a37008cdedefe8b046d0b40fbbe7f58101617 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 13:50:06 -0400 Subject: [PATCH 0411/1072] revert: remove unrelated rhaiis changes from this PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert agent_analysis.url change and workload rampup additions — these belong in a separate PR per reviewer feedback. Co-authored-by: Cursor --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- projects/rhaiis/orchestration/config.d/workloads.yaml | 4 ---- projects/rhaiis/orchestration/runtime_config.py | 3 --- projects/rhaiis/orchestration/test_phase.py | 2 -- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 574cbec1c..89b904e48 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -94,4 +94,4 @@ profiler: agent_analysis: enabled: false severity_threshold: 10 - url: "" + url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 833cbb5e0..1cee1e4c6 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -7,24 +7,20 @@ profile1: data: "prompt_tokens=1000,output_tokens=1000" rates: [1,50,100,200,300] max_seconds: 450 - rampup: 10 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] max_seconds: 450 - rampup: 10 profile3: data: "prompt_tokens=2048,output_tokens=128" rates: [1, 50, 100, 200, 300] max_seconds: 450 - rampup: 10 profile4: data: "prompt_tokens=8000,output_tokens=1000" rates: [1,25,50,75,100] max_seconds: 450 - rampup: 10 samples: 50 diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index 96d563fc3..3f898fa81 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -168,7 +168,6 @@ def build_guidellm_args( data: str, rates: list[int], max_seconds: int, - rampup: int | None = None, ) -> list[str]: guidellm_args = [] for key, value in benchmark_cfg.get("args", {}).items(): @@ -179,8 +178,6 @@ def build_guidellm_args( guidellm_args.append(f"--data={data}") guidellm_args.append(f"--rate={_format_arg_value(rates)}") guidellm_args.append(f"--max-seconds={max_seconds}") - if rampup is not None: - guidellm_args.append(f"--rampup={rampup}") return guidellm_args diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 185b63931..4bcc1c520 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -416,7 +416,6 @@ def _run_workload_benchmark( workload = runtime_config.get_workload(workload_key) rates = workload.get("rates", [1]) max_seconds = workload.get("max_seconds", 180) - rampup = workload.get("rampup") from projects.core.library import config from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( @@ -468,7 +467,6 @@ def _run_workload_benchmark( data=workload["data"], rates=rates, max_seconds=max_seconds, - rampup=rampup, ) run_guidellm_benchmark( From 1325e18f844475ef1cade34f3edcd0d5b407e2d8 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 13:58:16 -0400 Subject: [PATCH 0412/1072] refactor: address PR review comments on caliper layering and robustness - Create projects/caliper/public/file_export.py to expose engine functions; orchestration imports from public instead of engine - Refactor precreate_mlflow_run() to accept secrets_path, experiment, workspace from caller instead of reading project config - Combine precreate_mlflow_run() and write_mlflow_precreated_run_marker() into a single call - Refactor build_mlflow_run_url() to accept secrets_path and workspace from caller - Move config reads to rhaiis callers (test_phase.py, regression.py) - Save/restore tracking URI in finally block alongside MLFLOW_WORKSPACE - URL-encode workspace in MLflow URL query string - Add logger.warning for all failure return paths - Warn on invalid marker YAML object type - Remove bare except swallowing in _read_mlflow_ids_from_marker - Add docstring explaining MLFLOW_PRECREATED_RUN_MARKER purpose - Revert unrelated rhaiis changes (agent_analysis.url, workloads rampup) Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 180 ++++++++++---------- projects/caliper/public/__init__.py | 1 + projects/caliper/public/file_export.py | 17 ++ projects/rhaiis/orchestration/test_phase.py | 46 +++-- projects/rhaiis/postprocess/regression.py | 23 ++- 5 files changed, 164 insertions(+), 103 deletions(-) create mode 100644 projects/caliper/public/__init__.py create mode 100644 projects/caliper/public/file_export.py diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index d6652993e..ac1cb0b5d 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -266,42 +266,41 @@ def run_from_orchestration_config( return yaml.safe_load(f.read()) -def precreate_mlflow_run() -> dict[str, str]: - """Pre-create an MLflow run so its IDs can be embedded in test labels before CSV generation. +MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" +"""Marker file written by the test step to communicate the pre-created MLflow +run ID and experiment ID to the export step. + +The test step pre-creates an MLflow run (immediately ended) so that the +``run_id`` and ``experiment_id`` are known *before* CSV generation. These IDs +are embedded in test labels and flow into the dashboard CSV. The export step +later discovers the marker via :func:`_discover_precreated_mlflow_run_id` and +resumes the same run (``mlflow.start_run(run_id=...)``) to upload artifacts, +avoiding a duplicate run. + +The marker is a YAML file with keys ``run_id`` and ``experiment_id``, written +to ``ARTIFACT_DIR`` by :func:`precreate_mlflow_run`. +""" - The run is created and immediately ended (status FINISHED). The export step - will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. - Returns a dict with ``run_id`` and ``experiment_id`` on success, - or an empty dict if MLflow is unavailable. - """ - from projects.core.library import config +def precreate_mlflow_run( + *, + secrets_path: Path, + experiment: str | None = None, + workspace: str | None = None, +) -> dict[str, str]: + """Pre-create an MLflow run and write the marker file to ``ARTIFACT_DIR``. - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False - ) - if not vault_name or not vault_key: - logger.info("MLflow vault not configured, skipping run pre-creation") - return {} + The run is created and immediately ended (status FINISHED). The export step + will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - logger.info("MLflow secrets file not found, skipping run pre-creation") - return {} - - experiment = config.project.get_config( - "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False - ) - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) + The caller (test harness) is responsible for reading config and vault paths; + this function does not access the project config directly. + Returns a dict with ``run_id`` and ``experiment_id``. + """ import mlflow - from projects.caliper.engine.file_export.mlflow_secrets import ( + from projects.caliper.public.file_export import ( load_mlflow_secrets_yaml, mlflow_connection_env, ) @@ -310,6 +309,7 @@ def precreate_mlflow_run() -> dict[str, str]: tracking_uri = secrets_data.get("tracking_uri", "") prev_workspace = os.environ.get("MLFLOW_WORKSPACE") + prev_tracking_uri = mlflow.get_tracking_uri() try: with mlflow_connection_env(secrets_data): if tracking_uri: @@ -329,84 +329,77 @@ def precreate_mlflow_run() -> dict[str, str]: os.environ["MLFLOW_WORKSPACE"] = prev_workspace else: os.environ.pop("MLFLOW_WORKSPACE", None) + mlflow.set_tracking_uri(prev_tracking_uri) - logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) - - return { - "run_id": run_id, - "experiment_id": experiment_id, - } - - -MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" + meta = {"run_id": run_id, "experiment_id": experiment_id} - -def write_mlflow_precreated_run_marker(meta: dict[str, str]) -> None: - """Persist the pre-created MLflow run_id to disk so the export step can resume it.""" marker_path = env.ARTIFACT_DIR / MLFLOW_PRECREATED_RUN_MARKER marker_path.parent.mkdir(parents=True, exist_ok=True) marker_path.write_text(yaml.safe_dump(meta, sort_keys=False), encoding="utf-8") - logger.info("Wrote MLflow pre-created run marker: %s", marker_path) + logger.info( + "Pre-created MLflow run %s (experiment=%s), marker: %s", + run_id, + experiment_id, + marker_path, + ) + + return meta def _read_mlflow_ids_from_marker() -> tuple[str, str]: """Read run_id and experiment_id from the pre-created marker file on disk.""" - try: - artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None - if not artifact_dir: - return "", "" - for marker in sorted(artifact_dir.rglob(MLFLOW_PRECREATED_RUN_MARKER)): - data = yaml.safe_load(marker.read_text(encoding="utf-8")) - if isinstance(data, dict) and data.get("run_id"): - return data["run_id"], data.get("experiment_id", "") - except Exception: - pass + artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None + if not artifact_dir: + logger.warning("ARTIFACT_DIR not set, cannot read MLflow marker") + return "", "" + for marker in sorted(artifact_dir.rglob(MLFLOW_PRECREATED_RUN_MARKER)): + data = yaml.safe_load(marker.read_text(encoding="utf-8")) + if not isinstance(data, dict): + logger.warning( + "MLflow marker %s has unexpected type %s, skipping", marker, type(data).__name__ + ) + continue + if data.get("run_id"): + return data["run_id"], data.get("experiment_id", "") + logger.warning("No valid MLflow marker found under %s", artifact_dir) return "", "" -def build_mlflow_run_url() -> str: - """Construct the MLflow run URL at runtime from vault secrets and the marker file.""" - try: - from projects.caliper.engine.file_export.mlflow_secrets import ( - assert_tracking_uri_has_no_userinfo, - load_mlflow_secrets_yaml, - ) - from projects.core.library import config +def build_mlflow_run_url( + *, + secrets_path: Path, + workspace: str | None = None, +) -> str: + """Construct the MLflow run URL from vault secrets and the marker file. - run_id, experiment_id = _read_mlflow_ids_from_marker() - if not run_id or not experiment_id: - return "" + The caller (test harness) is responsible for resolving ``secrets_path`` + and ``workspace``; this function does not access the project config. + """ + from urllib.parse import quote - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", - None, - print=False, - warn=False, - ) - if not vault_name or not vault_key: - return "" + from projects.caliper.public.file_export import ( + assert_tracking_uri_has_no_userinfo, + load_mlflow_secrets_yaml, + ) - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - return "" + run_id, experiment_id = _read_mlflow_ids_from_marker() + if not run_id or not experiment_id: + logger.warning("Cannot build MLflow URL: run_id or experiment_id missing from marker") + return "" - secrets_data = load_mlflow_secrets_yaml(secrets_path) - tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") - if not tracking_uri.startswith(("http://", "https://")): - return "" - assert_tracking_uri_has_no_userinfo(tracking_uri) + if not secrets_path.exists(): + logger.warning("Cannot build MLflow URL: secrets file %s not found", secrets_path) + return "" - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) - qs = f"?workspace={workspace}" if workspace else "" - return f"{tracking_uri}/#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" - except Exception: - logger.debug("Failed to build MLflow run URL", exc_info=True) + secrets_data = load_mlflow_secrets_yaml(secrets_path) + tracking_uri = secrets_data.get("tracking_uri", "").rstrip("/") + if not tracking_uri.startswith(("http://", "https://")): + logger.warning("Cannot build MLflow URL: tracking_uri has unsupported scheme") return "" + assert_tracking_uri_has_no_userinfo(tracking_uri) + + qs = f"?workspace={quote(workspace, safe='')}" if workspace else "" + return f"{tracking_uri}/#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: @@ -414,7 +407,14 @@ def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: for marker in sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)): try: data = yaml.safe_load(marker.read_text(encoding="utf-8")) - run_id = data.get("run_id") if isinstance(data, dict) else None + if not isinstance(data, dict): + logger.warning( + "MLflow marker %s has unexpected type %s, skipping", + marker, + type(data).__name__, + ) + continue + run_id = data.get("run_id") if run_id: logger.info("Found pre-created MLflow run_id: %s (from marker %s)", run_id, marker) return run_id diff --git a/projects/caliper/public/__init__.py b/projects/caliper/public/__init__.py new file mode 100644 index 000000000..6ef3f2a54 --- /dev/null +++ b/projects/caliper/public/__init__.py @@ -0,0 +1 @@ +"""Public API surface for caliper — safe for orchestration-layer imports.""" diff --git a/projects/caliper/public/file_export.py b/projects/caliper/public/file_export.py new file mode 100644 index 000000000..a8c5723a6 --- /dev/null +++ b/projects/caliper/public/file_export.py @@ -0,0 +1,17 @@ +"""Public re-exports of caliper engine file_export utilities. + +Orchestration code should import from here instead of reaching into +``projects.caliper.engine.file_export`` directly. +""" + +from projects.caliper.engine.file_export.mlflow_secrets import ( + assert_tracking_uri_has_no_userinfo, + load_mlflow_secrets_yaml, + mlflow_connection_env, +) + +__all__ = [ + "assert_tracking_uri_has_no_userinfo", + "load_mlflow_secrets_yaml", + "mlflow_connection_env", +] diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 4bcc1c520..51fc53d7f 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -9,6 +9,7 @@ import yaml from projects.core.library import env +from projects.core.library import vault as vault_lib from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.rhaiis.orchestration import runtime_config @@ -296,23 +297,44 @@ def _run_test( _warnings.append("Profiler trace upload failed") # Pre-create MLflow run so its URL can be embedded in test labels / CSV - from projects.caliper.orchestration.export import ( - precreate_mlflow_run, - write_mlflow_precreated_run_marker, - ) - mlflow_run_meta: dict[str, str] = {} try: - mlflow_run_meta = precreate_mlflow_run() + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", + None, + print=False, + warn=False, + ) + if vault_name and vault_key: + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if secrets_path and secrets_path.exists(): + from projects.caliper.orchestration.export import precreate_mlflow_run + + mlflow_run_meta = precreate_mlflow_run( + secrets_path=secrets_path, + experiment=config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", + None, + print=False, + warn=False, + ), + workspace=config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", + None, + print=False, + warn=False, + ), + ) + else: + logger.info("MLflow secrets file not found, skipping run pre-creation") + else: + logger.info("MLflow vault not configured, skipping run pre-creation") except Exception: logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) - if mlflow_run_meta.get("run_id"): - try: - write_mlflow_precreated_run_marker(mlflow_run_meta) - except Exception: - logger.warning("Failed to write MLflow marker file; continuing", exc_info=True) - # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None for wl_key in workload_keys: diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 565da1af1..e93b84edf 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -335,8 +335,29 @@ def run_regression_analysis( def _build_mlflow_run_url() -> str: """Construct the MLflow run URL at runtime from vault secrets and config.""" from projects.caliper.orchestration.export import build_mlflow_run_url + from projects.core.library import config + from projects.core.library import vault as vault_lib - return build_mlflow_run_url() + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False + ) + if not vault_name or not vault_key: + logger.warning("Cannot build MLflow URL: vault not configured") + return "" + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.warning("Cannot build MLflow URL: secrets file not found") + return "" + + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + + return build_mlflow_run_url(secrets_path=secrets_path, workspace=workspace) PROFILE_DISPLAY_NAMES = { From 2b951ca2f616ab039c66855290cadeddd0e168fd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 15:05:34 -0400 Subject: [PATCH 0413/1072] fix: send success notification regardless of dashboard CSV config The slack_notify_always notification was inside _sync_postprocessed_dashboard_csv() which returns early when csv_dashboard.enabled is false. Move to a standalone function called from run() so it fires independently. Co-authored-by: Cursor --- projects/rhaiis/orchestration/test_phase.py | 59 +++++++++++++-------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 51fc53d7f..232938c68 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -54,6 +54,9 @@ def run( logger.exception("Dashboard CSV S3 sync after postprocessing failed") ret = 1 + if ret == 0: + _maybe_send_success_notification(model_key, workload_keys) + return ret @@ -663,30 +666,42 @@ def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) ) return - if config.project.get_config("tests.rhaiis.slack_notify_always", False): - from projects.rhaiis.postprocess.regression import send_success_notification - model_cfg = runtime_config.get_model(model_key) - accelerator = runtime_config.get_accelerator() - engine = runtime_config.get_engine() - engine_defaults = runtime_config.get_engine_args(engine) - first_workload = runtime_config.get_workload(workload_keys[0]) - ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) - tp = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or "" - dp = ea.get("data-parallel-size") or ea.get("dp-size") or "" +def _maybe_send_success_notification(model_key: str, workload_keys: list[str]) -> None: + """Send a Slack success notification if slack_notify_always is set and no regression check.""" + from projects.core.library import config - send_success_notification( - model=model_cfg.get("hf_model_id", ""), - accelerator=accelerator, - job_id=os.environ.get("FJOB_NAME", ""), - slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), - notification_vault="psap-forge-notifications", - tp=str(tp), - dp=str(dp), - version=version, - workload_keys=workload_keys, - cluster=config.project.get_config("rhaiis.cluster_tag", ""), - ) + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if compare_version: + return + + if not config.project.get_config("tests.rhaiis.slack_notify_always", False): + return + + from projects.rhaiis.postprocess.regression import send_success_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) + first_workload = runtime_config.get_workload(workload_keys[0]) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) + tp = ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or "" + dp = ea.get("data-parallel-size") or ea.get("dp-size") or "" + version = config.project.get_config("tests.rhaiis.version", "") + + send_success_notification( + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(tp), + dp=str(dp), + version=version, + workload_keys=workload_keys, + cluster=config.project.get_config("rhaiis.cluster_tag", ""), + ) def _upload_predictor_log(run_uuid: str) -> None: From a5c658d682d89f270b3f326160cd5e032392168b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 15:11:19 -0400 Subject: [PATCH 0414/1072] fix: log warning instead of silently swallowing exception in send_success_notification Co-authored-by: Cursor --- projects/rhaiis/postprocess/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index e93b84edf..6d4096b4f 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -603,7 +603,7 @@ def send_success_notification( ) dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" except Exception: - pass + logger.warning("Failed to build dashboard URL for notification", exc_info=True) mlflow_url = _build_mlflow_run_url() mlflow_line = f"*MLflow:* <{mlflow_url}|View Run>\n" if mlflow_url else "" From 01d220fa540f1e86b9fbc37208327e1178301a91 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 15:49:16 -0400 Subject: [PATCH 0415/1072] revert .gitignore to upstream state Move personal ignore patterns (fournos-job-*.yaml, kubeconfig*) to .git/info/exclude per reviewer feedback. Co-authored-by: Cursor --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 565ad2d5b..80ea2d510 100644 --- a/.gitignore +++ b/.gitignore @@ -162,7 +162,4 @@ Thumbs.db temp/ # FORGE launcher configuration (personal settings) -projects/core/launcher/launcher_config.yaml - -fournos-job-*.yaml -kubeconfig* \ No newline at end of file +projects/core/launcher/launcher_config.yaml \ No newline at end of file From 3af7088edcc7d4285890313c7b7a575c5cffc525 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 4 Aug 2026 19:24:52 -0400 Subject: [PATCH 0416/1072] rename FAILURE/FAILURES artifacts to .txt for MLflow preview MLflow cannot preview files without a recognized extension. Adding .txt lets operators read failure details directly in the MLflow UI instead of downloading each file. Co-authored-by: Cursor --- projects/cluster/toolbox/set_scale/main.py | 2 +- projects/core/agentic/artifact_processing.py | 4 ++-- projects/core/agentic/on_failure/__init__.py | 2 +- projects/core/ci_entrypoint/prepare_ci.py | 10 +++++----- projects/core/dsl/runtime.py | 2 +- projects/core/library/ci.py | 2 +- projects/core/notifications/send.py | 6 +++--- .../guidellm/toolbox/run_guidellm_benchmark/main.py | 2 +- projects/legacy/library/ansible_toolbox.py | 2 +- projects/legacy/library/env.py | 2 +- projects/matrix_benchmarking/library/matbenchmark.py | 2 +- projects/matrix_benchmarking/library/visualize.py | 6 +++--- .../subproject/matrix_benchmarking/analyze_lts.py | 2 +- 13 files changed, 22 insertions(+), 22 deletions(-) diff --git a/projects/cluster/toolbox/set_scale/main.py b/projects/cluster/toolbox/set_scale/main.py index b4d444241..29b204bc4 100755 --- a/projects/cluster/toolbox/set_scale/main.py +++ b/projects/cluster/toolbox/set_scale/main.py @@ -431,7 +431,7 @@ def wait_for_compute_nodes_ready(args, ctx): failure_content += f"**Parent:** machineset/{ctx.target_machineset}\n" failure_content += f"**Error:** {detail['error']}\n\n" - failure_file = args.artifact_dir / "FAILURE" + failure_file = args.artifact_dir / "FAILURE.txt" with open(failure_file, "w") as f: f.write(failure_content) diff --git a/projects/core/agentic/artifact_processing.py b/projects/core/agentic/artifact_processing.py index 651cee6b2..a405ec85c 100644 --- a/projects/core/agentic/artifact_processing.py +++ b/projects/core/agentic/artifact_processing.py @@ -144,7 +144,7 @@ def find_failure_files(base_artifact_dir: Path) -> list[Path]: """ failure_files = [] - for failure_file in base_artifact_dir.rglob("FAILURE"): + for failure_file in base_artifact_dir.rglob("FAILURE.txt"): if not failure_file.exists(): continue if failure_file.parent == base_artifact_dir: @@ -153,7 +153,7 @@ def find_failure_files(base_artifact_dir: Path) -> list[Path]: logger.debug(f"Found FAILURE file: {failure_file}") if not failure_files: - base_failure = base_artifact_dir / "FAILURE" + base_failure = base_artifact_dir / "FAILURE.txt" if base_failure.exists(): failure_files.append(base_failure) diff --git a/projects/core/agentic/on_failure/__init__.py b/projects/core/agentic/on_failure/__init__.py index be330aaf1..ef054bbde 100755 --- a/projects/core/agentic/on_failure/__init__.py +++ b/projects/core/agentic/on_failure/__init__.py @@ -427,7 +427,7 @@ def analyze_single_failure_multi_query( file_lower = file_path.lower() # Skip the main failure file and task.log as they're already included - if file_path.endswith("FAILURE") or file_path.endswith("task.log"): + if file_path.endswith("FAILURE.txt") or file_path.endswith("task.log"): continue if any(pattern in file_lower for pattern in high_priority_patterns): diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 11dad7411..d8ebfa7c3 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -367,7 +367,7 @@ def system_prechecks() -> bool: artifact_path = Path(artifact_dir) # Check for existing failures - failures_file = artifact_path / "FAILURES" + failures_file = artifact_path / "FAILURES.txt" if failures_file.exists() and not os.environ.get("FORGE_IGNORE_FAILURES_FILE"): raise ValueError( f"File '{failures_file}' already exists, cannot continue. Set FORGE_IGNORE_FAILURES_FILE=1 to ignore this." @@ -386,7 +386,7 @@ def system_prechecks() -> bool: os.environ["FORGE_OPENSHIFT_CI_STEP_DIR"] = step_dir # Remove any old failure markers - old_failure = artifact_path / "FAILURE" + old_failure = artifact_path / "FAILURE.txt" if old_failure.exists(): old_failure.unlink() @@ -801,8 +801,8 @@ def postchecks( pass elif finish_reason == FinishReason.ERROR: # Find all FAILURE files and consolidate them - failure_files = list(artifact_path.glob("**/FAILURE")) - failures_file = artifact_path / "FAILURES" + failure_files = list(artifact_path.glob("**/FAILURE.txt")) + failures_file = artifact_path / "FAILURES.txt" with failures_file.open("w") as f: for failure_file in sorted(failure_files): @@ -822,7 +822,7 @@ def postchecks( duration_str = generate_duration_and_timing_file(start_time, artifact_path) # Check if there were failures - failures_file = artifact_path / "FAILURES" + failures_file = artifact_path / "FAILURES.txt" if finish_reason != FinishReason.SUCCESS or ( failures_file.exists() and failures_file.stat().st_size > 0 ): diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index c0c4b8b88..50cebece9 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -665,7 +665,7 @@ def _generate_failure_file_for_agent(exception: Exception) -> None: logger.warning("No ARTIFACT_DIR available - cannot generate FAILURE file for agent") return - failure_file_path = artifact_dir / "FAILURE" + failure_file_path = artifact_dir / "FAILURE.txt" # Generate failure content failure_content = _format_failure_content_for_agent(exception) diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 894e5c7d4..d58ec9e04 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -124,7 +124,7 @@ def _display_error_summary(e: Exception) -> None: def _write_error_summary_to_file(summary_lines: list) -> None: """Write the error summary to FAILURE file.""" - failures_file = env.ARTIFACT_DIR / "FAILURE" + failures_file = env.ARTIFACT_DIR / "FAILURE.txt" try: content = "\n".join(summary_lines) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index ab6f64acc..04a9b8f61 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -214,7 +214,7 @@ def _get_notification_content(artifact_dir: pathlib.Path, get_link, get_bold) -> Formatted notification content string """ notifications_dir = ci_lib.get_ci_metadata_dir() / "notifications" - failures_file = artifact_dir / "FAILURES" + failures_file = artifact_dir / "FAILURES.txt" # Guard: Check if notifications directory exists if not notifications_dir.exists(): @@ -279,7 +279,7 @@ def _get_notification_content(artifact_dir: pathlib.Path, get_link, get_bold) -> # Add link to FAILURES file if it exists (but don't include content) if failures_file.exists(): content += f""" -• {get_link("Raw failure details", "FAILURES", is_raw_file=True)} +• {get_link("Raw failure details", "FAILURES.txt", is_raw_file=True)} """ return content @@ -318,7 +318,7 @@ def _get_fallback_failure_content(failures_file: pathlib.Path, get_link, get_bol head = DEFAULT_HEAD return f""" -• {get_link("Failure indicator", "FAILURES", is_raw_file=True)}: +• {get_link("Failure indicator", "FAILURES.txt", is_raw_file=True)}: ``` {"".join(lines[:head])} {"[...]" if len(lines) > head else ""} diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 5457c5487..af3501f57 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -260,7 +260,7 @@ def wait_guidellm_benchmark_task(args, ctx): return f"GuideLLM benchmark {ctx.benchmark_name} completed" if failed: # Write failure file - failure_file = args.artifact_dir / "FAILURE" + failure_file = args.artifact_dir / "FAILURE.txt" failure_message = f"""GuideLLM benchmark job '{ctx.benchmark_name}' failed. Check the job logs for detailed error information: diff --git a/projects/legacy/library/ansible_toolbox.py b/projects/legacy/library/ansible_toolbox.py index 799f8066d..5e8fbfcf0 100644 --- a/projects/legacy/library/ansible_toolbox.py +++ b/projects/legacy/library/ansible_toolbox.py @@ -397,7 +397,7 @@ def _run(self): if ret != 0: extra_dir_name = pathlib.Path(env["ARTIFACT_EXTRA_LOGS_DIR"]).name - with open(artifact_extra_logs_dir / "FAILURE", "a") as f: + with open(artifact_extra_logs_dir / "FAILURE.txt", "a") as f: print(f"[{extra_dir_name}] {' '.join(sys.argv)} --> {ret}", file=f) raise SystemExit(ret) diff --git a/projects/legacy/library/env.py b/projects/legacy/library/env.py index f18b0b2e9..1c58f0dca 100644 --- a/projects/legacy/library/env.py +++ b/projects/legacy/library/env.py @@ -100,7 +100,7 @@ def __exit__(self, ex_type, ex_value, exc_traceback): if ex_value: logging.error(f"Caught exception {ex_type.__name__}: {ex_value}") - with open(get_tls_artifact_dir() / "FAILURE", "a") as f: + with open(get_tls_artifact_dir() / "FAILURE.txt", "a") as f: print(f"{ex_type.__name__}: {ex_value}", file=f) print( "".join(traceback.format_exception(None, value=ex_value, tb=exc_traceback)), diff --git a/projects/matrix_benchmarking/library/matbenchmark.py b/projects/matrix_benchmarking/library/matbenchmark.py index ecb2cc889..0737e9dd6 100644 --- a/projects/matrix_benchmarking/library/matbenchmark.py +++ b/projects/matrix_benchmarking/library/matbenchmark.py @@ -126,7 +126,7 @@ def run_benchmark(args, dry_run=False): except Exception: msg = "MatrixBenchmark benchmark failed." logging.error(msg) - with open(env.ARTIFACT_DIR / "FAILURE", "w") as f: + with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: print(msg, file=f) return True # failed diff --git a/projects/matrix_benchmarking/library/visualize.py b/projects/matrix_benchmarking/library/visualize.py index f24181948..2b4385e39 100755 --- a/projects/matrix_benchmarking/library/visualize.py +++ b/projects/matrix_benchmarking/library/visualize.py @@ -328,7 +328,7 @@ def log_has_errors(log_file): if not has_errors: continue - with open(env.ARTIFACT_DIR / "FAILURE", "a") as fail_f: + with open(env.ARTIFACT_DIR / "FAILURE.txt", "a") as fail_f: print(line, end="", file=fail_f) logging.error(line.strip()) @@ -389,7 +389,7 @@ def generate_visualization( if fatal_errors: msg = f"A fatal error happened during the results parsing, aborting the visualization ({', '.join(fatal_errors)})." - with open(env.ARTIFACT_DIR / "FAILURE", "w") as f: + with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: print(msg, file=f) logging.error(msg) raise RuntimeError(msg) @@ -543,7 +543,7 @@ def generate_visualization( # if None, ignore logging.error(msg) - with open(env.ARTIFACT_DIR / "FAILURE", "w") as f: + with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: print(msg, file=f) raise RuntimeError(msg) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py index d613c07c2..9005525ea 100644 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py +++ b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py @@ -99,7 +99,7 @@ def run(): logging.info(f"The regression analyze finished with code {failures}.") except Exception as e: - with open(store_dir / "FAILURE", "a") as f: + with open(store_dir / "FAILURE.txt", "a") as f: print(str(e), file=f) import traceback From ee9aeecb79b74c2f0e9622b117cf39a93fcf021d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 14:35:01 +0200 Subject: [PATCH 0417/1072] [caliper] Add label include/exclude filtering for all the entrypoints --- projects/caliper/cli/commands.py | 2 ++ projects/caliper/engine/kpi/generate.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 3ad22c871..6b64b8599 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -575,6 +575,8 @@ def kpi_generate( use_cache=True, cache_path=None, format_type=format_type, + include_label_filter=include_filter, + exclude_label_filter=exclude_filter, ) status_data = { "success": True, diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index 99abacada..a80e3cc29 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -19,6 +19,8 @@ def run_kpi_generate( use_cache: bool, cache_path: Path | None, format_type: str = "hierarchical", + include_label_filter: list[tuple[str, str]] | None = None, + exclude_label_filter: list[tuple[str, str]] | None = None, ) -> list[dict[str, Any]]: """Generate KPI output in specified format. @@ -30,6 +32,8 @@ def run_kpi_generate( use_cache: Whether to use cached parse results cache_path: Path to cache file (not used currently) format_type: Output format - "hierarchical" (default) or "jsonl" + include_label_filter: Include only directories matching these label filters + exclude_label_filter: Exclude directories matching these label filters Returns: List of KPI records @@ -39,6 +43,8 @@ def run_kpi_generate( plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, + include_label_filter=include_label_filter, + exclude_label_filter=exclude_label_filter, ) compute = plugin.compute_kpis rows: list[dict[str, Any]] = compute(model) From 34d299452b1ebc343b91e4227fc6943c0584f962 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 3 Aug 2026 16:42:57 +0200 Subject: [PATCH 0418/1072] [caliper] orchestration: postprocess: fail if no test result is dectected --- projects/caliper/orchestration/postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index b8297a42a..3f74ed646 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -692,7 +692,7 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A warning_msg = result.get("message", "unknown warning") logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") elif status == "failed": - error_msg = result.get("message", "unknown error") + error_msg = result.get("error", result.get("message", "unknown error")) traceback_msg = result.get("traceback") logger.error(f"Step '{step_name}' failed: {error_msg}") if traceback_msg: From 8d3abcf2d8082c448a9129faf1e18868e387010e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 11:32:33 +0200 Subject: [PATCH 0419/1072] [caliper] engine: kpi: analyze: fail on configuration load --- projects/caliper/engine/kpi/analyze.py | 63 ++++++++++++++++++++------ 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index ba0bfa0ae..667a3522b 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -59,29 +59,51 @@ class AnalysisReport: def _load_analysis_config(plugin_module: str) -> AnalysisConfig: - """Load analysis config from plugin module if available, else use defaults. + """Load analysis config from plugin module. - Plugins can expose an `analysis_config` dict or `get_analysis_config()` callable. + Plugins must expose an `analysis_config` dict or `get_analysis_config()` callable. + Raises ValueError if config is not available. """ try: mod = __import__(plugin_module, fromlist=[""]) - if hasattr(mod, "get_analysis_config"): + except ImportError as exc: + raise ValueError( + f"Failed to import plugin module '{plugin_module}': {exc}. " + f"Check that the plugin module exists and is importable." + ) from exc + + if hasattr(mod, "get_analysis_config"): + try: raw = mod.get_analysis_config() - elif hasattr(mod, "analysis_config"): - raw = mod.analysis_config - else: - return AnalysisConfig() + except Exception as exc: + raise ValueError( + f"Plugin module '{plugin_module}' has get_analysis_config() but calling it failed: {exc}" + ) from exc + elif hasattr(mod, "analysis_config"): + raw = mod.analysis_config + else: + raise ValueError( + f"Plugin module '{plugin_module}' is missing required analysis configuration. " + f"Plugin must provide either 'analysis_config' attribute or 'get_analysis_config()' function." + ) - if isinstance(raw, AnalysisConfig): - return raw - if isinstance(raw, dict): + if isinstance(raw, AnalysisConfig): + return raw + if isinstance(raw, dict): + try: return AnalysisConfig( **{k: v for k, v in raw.items() if k in AnalysisConfig.__dataclass_fields__} ) - except (ImportError, Exception) as exc: - logger.debug("Could not load analysis config from plugin %s: %s", plugin_module, exc) - - return AnalysisConfig() + except Exception as exc: + raise ValueError( + f"Plugin module '{plugin_module}' analysis config has invalid format: {exc}. " + f"Config must be a dict with valid AnalysisConfig fields or an AnalysisConfig instance." + ) from exc + else: + raise ValueError( + f"Plugin module '{plugin_module}' analysis config has unsupported type '{type(raw).__name__}'. " + f"Must be a dict or AnalysisConfig instance." + ) def _match_key( @@ -286,7 +308,18 @@ def run_kpi_analysis( "completed_at": time.time(), } - config = _load_analysis_config(plugin_module) + try: + config = _load_analysis_config(plugin_module) + except ValueError as exc: + logger.error("Failed to load analysis config: %s", exc) + return { + "status": "failed", + "success": False, + "error": f"Analysis configuration error: {exc}", + "exit_code": 1, + "completed_at": time.time(), + } + logger.info( " config: comparison_keys=%s, ignored_keys=%s", config.comparison_keys, From 3bcf4418ac37573db01991c140345d6485e43a25 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 14:56:50 +0200 Subject: [PATCH 0420/1072] [caliper] cli: s3_export: use the Caliper config to know what to export --- projects/caliper/cli/s3_export.py | 95 +++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index 5c500ea55..91fd5eb50 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -86,63 +86,104 @@ def list_ai_data_files(ai_data_dir: Path) -> list[Path]: return files -def list_csv_files(output_dir: Path) -> list[Path]: +def list_csv_files(output_dir: Path, postprocess_config=None) -> list[Path]: """List CSV files in the postprocess output directory. Args: output_dir: Path to postprocess output directory + postprocess_config: Postprocess configuration to determine CSV file path Returns: List of CSV file paths to upload """ - csv_files = list(output_dir.glob("*.csv")) + csv_files = [] - logger.info(f"Found {len(csv_files)} CSV files in output directory: {output_dir}") - for csv_file in csv_files: - logger.info(f" - {csv_file.name} ({csv_file.stat().st_size} bytes)") + if postprocess_config and postprocess_config.kpi.csv.enabled: + # Use configured CSV file path + csv_file_path = output_dir / postprocess_config.kpi.csv.output + if csv_file_path.exists(): + csv_files.append(csv_file_path) + logger.info( + f"Found configured CSV file: {csv_file_path.name} ({csv_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured CSV file not found: {csv_file_path}") + else: + # Fallback to glob pattern for backward compatibility + csv_files = list(output_dir.glob("*.csv")) + logger.info(f"Found {len(csv_files)} CSV files in output directory: {output_dir}") + for csv_file in csv_files: + logger.info(f" - {csv_file.name} ({csv_file.stat().st_size} bytes)") return csv_files -def list_kpi_json_files(output_dir: Path) -> list[Path]: +def list_kpi_json_files(output_dir: Path, postprocess_config=None) -> list[Path]: """List KPI JSON files in the postprocess output directory. Args: output_dir: Path to postprocess output directory + postprocess_config: Postprocess configuration to determine KPI JSON file path Returns: List of KPI JSON file paths to upload """ - kpi_json_files = list(output_dir.glob("kpis.json")) + kpi_json_files = [] - logger.info(f"Found {len(kpi_json_files)} KPI JSON files in output directory: {output_dir}") - for kpi_file in kpi_json_files: - logger.info(f" - {kpi_file.name} ({kpi_file.stat().st_size} bytes)") + if postprocess_config and postprocess_config.kpi.artifacts_to_kpis.enabled: + # Use configured KPI JSON file path + kpi_file_path = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + if kpi_file_path.exists(): + kpi_json_files.append(kpi_file_path) + logger.info( + f"Found configured KPI JSON file: {kpi_file_path.name} ({kpi_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured KPI JSON file not found: {kpi_file_path}") + else: + # Fallback to glob pattern for backward compatibility + kpi_json_files = list(output_dir.glob("kpis.json")) + logger.info(f"Found {len(kpi_json_files)} KPI JSON files in output directory: {output_dir}") + for kpi_file in kpi_json_files: + logger.info(f" - {kpi_file.name} ({kpi_file.stat().st_size} bytes)") return kpi_json_files -def list_analysis_files(output_dir: Path) -> list[Path]: +def list_analysis_files(output_dir: Path, postprocess_config=None) -> list[Path]: """List all analysis output files in the output directory. Args: output_dir: Path to search for analysis files + postprocess_config: Postprocess configuration to determine analysis file path Returns: List of analysis file paths to upload """ files = [] - # Look for analysis output files (kpi_analyze.json is the most common) - analysis_patterns = ["kpi_analyze.json", "kpi_analysis.json", "*analyze*.json"] - for pattern in analysis_patterns: - for file_path in output_dir.glob(pattern): - if file_path.is_file(): - files.append(file_path) + if postprocess_config and postprocess_config.analyze.enabled: + # Use configured analysis file path + analysis_file_path = output_dir / postprocess_config.analyze.output + if analysis_file_path.exists(): + files.append(analysis_file_path) + logger.info( + f"Found configured analysis file: {analysis_file_path.name} ({analysis_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured analysis file not found: {analysis_file_path}") + else: + # Fallback to glob patterns for backward compatibility + analysis_patterns = ["kpi_analyze.json", "kpi_analysis.json", "*analyze*.json"] - logger.info(f"Found {len(files)} analysis files in output directory: {output_dir}") - for file_path in files: - logger.info(f" - {file_path.name} ({file_path.stat().st_size} bytes)") + for pattern in analysis_patterns: + for file_path in output_dir.glob(pattern): + if file_path.is_file(): + files.append(file_path) + + logger.info(f"Found {len(files)} analysis files in output directory: {output_dir}") + for file_path in files: + logger.info(f" - {file_path.name} ({file_path.stat().st_size} bytes)") return files @@ -659,10 +700,14 @@ def run_s3_export( # List local files to upload upload_files = [] - # Use directory searching as fallback for legacy orchestration calls - csv_files = list_csv_files(output_dir) if s3_config.include_csv else [] - kpi_json_files = list_kpi_json_files(output_dir) if s3_config.include_kpis_json else [] - analysis_files = list_analysis_files(output_dir) + # Use postprocess config to determine file locations + csv_files = list_csv_files(output_dir, postprocess_config) if s3_config.include_csv else [] + kpi_json_files = ( + list_kpi_json_files(output_dir, postprocess_config) + if s3_config.include_kpis_json + else [] + ) + analysis_files = list_analysis_files(output_dir, postprocess_config) upload_files.extend(csv_files) upload_files.extend(kpi_json_files) @@ -775,7 +820,7 @@ def run_s3_export( # Collect KPI JSON file details if s3_config.include_kpis_json: - kpi_json_files = list_kpi_json_files(output_dir) + kpi_json_files = list_kpi_json_files(output_dir, postprocess_config) for kpi_file in kpi_json_files: try: relative_kpi_path = str(kpi_file.relative_to(artifact_dir)) From 568abde3f8091ee12c61924d5454d5c69245ea87 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 15:28:57 +0200 Subject: [PATCH 0421/1072] [guidellm] postprocess: guidellm: plugin: configure the regression analyze --- projects/guidellm/postprocess/guidellm/plugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index f570050df..d58a700ab 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +from projects.caliper.engine.kpi.analyze import AnalysisConfig from projects.caliper.engine.model import ( ParseResult, PostProcessingPlugin, @@ -21,6 +22,16 @@ logger = logging.getLogger(__name__) +# Analysis configuration for KPI regression testing +analysis_config = AnalysisConfig( + comparison_keys=["product_version", "model_name"], + ignored_keys=[], + sorting_keys=["product_version"], + max_relative_regression=0.15, # 15% threshold (LLM performance can vary more) + min_baseline_points=2, # Require at least 2 baseline points for reliability +) + + # Plot registry - maps report names to their generator functions and parameters PLOT_REGISTRY = { "report_performance_analysis": { From bd3ab6b9d4d4e8d0aee4eedf3b1859645f36836c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 14:58:04 +0200 Subject: [PATCH 0422/1072] docs: caliper: plugin_regression: document Caliper plugin regression configuration --- docs/caliper/plugin_regression.md | 319 ++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 docs/caliper/plugin_regression.md diff --git a/docs/caliper/plugin_regression.md b/docs/caliper/plugin_regression.md new file mode 100644 index 000000000..35a19476e --- /dev/null +++ b/docs/caliper/plugin_regression.md @@ -0,0 +1,319 @@ +# KPI Regression Analysis Configuration + +This document explains how to configure KPI regression analysis in Caliper plugins. Regression analysis compares current KPI values against historical baselines to detect performance regressions. + +## Overview + +KPI regression analysis helps you: +- **Detect Performance Regressions**: Automatically flag when KPIs deteriorate compared to historical baselines +- **Track Performance Trends**: Compare current results against previous versions or configurations +- **Set Quality Gates**: Define acceptable thresholds for performance changes +- **Generate Reports**: Produce detailed analysis reports with regression verdicts + +## Analysis Configuration + +### AnalysisConfig Fields + +Every plugin **must** provide an analysis configuration that specifies how regression testing should be performed: + +```python +@dataclass +class AnalysisConfig: + """Configuration for KPI regression analysis.""" + + comparison_keys: list[str] = field(default_factory=list) + ignored_keys: list[str] = field(default_factory=list) + sorting_keys: list[str] = field(default_factory=list) + max_relative_regression: float = 0.1 + min_baseline_points: int = 1 +``` + +#### Field Descriptions + +**`comparison_keys`** *(required)* +- Label keys that define what we compare against +- Records must differ on at least one comparison key to be distinct baselines +- Example: `["version"]` means we test the current version against other versions +- Example: `["model", "version"]` means we compare across different model versions + +**`ignored_keys`** *(optional)* +- Label keys excluded when matching current to baseline records +- Allows testing across environmental differences +- Example: `["os", "hostname"]` means we match across different operating systems and machines +- Example: `["timestamp", "build_id"]` means we ignore timing and build-specific labels + +**`sorting_keys`** *(optional)* +- Label keys used to order entries in the output report +- Helps organize results in a logical sequence +- Example: `["platform", "model"]` sorts results by platform, then model + +**`max_relative_regression`** *(default: 0.1)* +- Fraction threshold for flagging regression (0.1 = 10%) +- Regressions exceeding this threshold are marked as failures +- Example: 0.05 for 5% threshold, 0.2 for 20% threshold + +**`min_baseline_points`** *(default: 1)* +- Minimum number of baseline data points required to run a test +- KPIs with fewer baselines are skipped +- Higher values require more historical data for reliable comparison + +## Providing Configuration in Plugins + +Plugins must provide analysis configuration using one of two methods: + +### Method 1: Static Configuration Attribute + +Provide a static `analysis_config` attribute in your plugin module: + +```python +# In your plugin module (e.g., projects/myproject/postprocess/myplugin/__init__.py) + +from projects.caliper.engine.kpi.analyze import AnalysisConfig + +# Static configuration +analysis_config = AnalysisConfig( + comparison_keys=["version"], + ignored_keys=["os", "hostname"], + sorting_keys=["platform", "model"], + max_relative_regression=0.05, # 5% threshold + min_baseline_points=3, +) +``` + +### Method 2: Dynamic Configuration Function + +Provide a `get_analysis_config()` function for dynamic configuration: + +```python +# In your plugin module +from projects.caliper.engine.kpi.analyze import AnalysisConfig + + +def get_analysis_config() -> AnalysisConfig: + """Return analysis configuration based on environment or settings.""" + + # Example: Different thresholds based on environment + import os + + if os.environ.get("CI_ENVIRONMENT") == "production": + regression_threshold = 0.02 # Stricter in production + else: + regression_threshold = 0.1 # More lenient in development + + return AnalysisConfig( + comparison_keys=["version", "model"], + ignored_keys=["hostname", "timestamp"], + sorting_keys=["version"], + max_relative_regression=regression_threshold, + min_baseline_points=2, + ) +``` + +### Method 3: Dictionary Configuration + +You can also provide configuration as a dictionary (automatically converted to AnalysisConfig): + +```python +# Dictionary format (converted automatically) +analysis_config = { + "comparison_keys": ["version"], + "ignored_keys": ["os"], + "sorting_keys": ["platform"], + "max_relative_regression": 0.1, + "min_baseline_points": 1, +} +``` + +## Configuration Examples + +### Example 1: Version-Based Comparison + +Test current version against previous versions, ignoring environmental differences: + +```python +analysis_config = AnalysisConfig( + comparison_keys=["version"], # Compare against different versions + ignored_keys=["os", "hostname"], # Ignore environment differences + sorting_keys=["platform"], # Sort results by platform + max_relative_regression=0.1, # 10% regression threshold + min_baseline_points=2, # Require at least 2 baseline points +) +``` + +### Example 2: Model Performance Comparison + +Compare different model configurations: + +```python +analysis_config = AnalysisConfig( + comparison_keys=["model", "version"], # Compare across model versions + ignored_keys=["timestamp", "run_id"], # Ignore timing/execution metadata + sorting_keys=["model", "version"], # Sort by model, then version + max_relative_regression=0.05, # Stricter 5% threshold + min_baseline_points=3, # Need more baselines for reliability +) +``` + +### Example 3: Multi-Platform Analysis + +Test across different platforms while tracking version changes: + +```python +analysis_config = AnalysisConfig( + comparison_keys=["version"], # Compare versions + ignored_keys=["hostname"], # Ignore specific machines + sorting_keys=["platform", "version"], # Group by platform, sort by version + max_relative_regression=0.15, # 15% threshold (platforms vary more) + min_baseline_points=1, # Accept single baseline per platform +) +``` + +### Example 4: Environment-Sensitive Configuration + +Different thresholds for different environments: + +```python +def get_analysis_config() -> AnalysisConfig: + import os + + env = os.environ.get("TEST_ENVIRONMENT", "dev") + + if env == "production": + return AnalysisConfig( + comparison_keys=["version"], + ignored_keys=["hostname"], + max_relative_regression=0.02, # Very strict for production + min_baseline_points=5, # Need lots of baselines + ) + elif env == "staging": + return AnalysisConfig( + comparison_keys=["version", "branch"], + ignored_keys=["hostname", "timestamp"], + max_relative_regression=0.05, # Moderate threshold + min_baseline_points=2, + ) + else: # development + return AnalysisConfig( + comparison_keys=["version"], + ignored_keys=["hostname", "timestamp", "developer"], + max_relative_regression=0.2, # Lenient for development + min_baseline_points=1, + ) +``` + +## Understanding Regression Logic + +### How Matching Works + +1. **Baseline Selection**: Records with different `comparison_keys` become potential baselines +2. **Label Matching**: Current and baseline records must match on all labels except: + - Labels in `ignored_keys` (explicitly ignored) + - Labels in `comparison_keys` (expected to differ) +3. **Threshold Comparison**: If relative change exceeds `max_relative_regression`: + - For "higher is better" KPIs: negative change > threshold = regression + - For "lower is better" KPIs: positive change > threshold = regression + +### Example Matching Scenario + +Given this configuration: +```python +analysis_config = AnalysisConfig( + comparison_keys=["version"], + ignored_keys=["hostname"], + max_relative_regression=0.1, +) +``` + +**Current Record**: +```json +{ + "kpi_id": "throughput", + "value": 90.0, + "labels": {"version": "v2.0", "platform": "GPU", "hostname": "worker-1"} +} +``` + +**Potential Baselines**: +```json +[ + { + "kpi_id": "throughput", + "value": 100.0, + "labels": {"version": "v1.9", "platform": "GPU", "hostname": "worker-2"} + }, + { + "kpi_id": "throughput", + "value": 85.0, + "labels": {"version": "v1.8", "platform": "GPU", "hostname": "worker-1"} + } +] +``` + +**Analysis**: +- Both baselines match: same `platform`, different `version`, `hostname` ignored +- Baseline mean: (100.0 + 85.0) / 2 = 92.5 +- Relative change: (90.0 - 92.5) / 92.5 = -0.027 (-2.7%) +- Result: No regression (2.7% < 10% threshold) + +## Error Handling + +If your plugin is missing analysis configuration, Caliper will fail with detailed error messages: + +``` +Analysis configuration error: Plugin module 'myproject.postprocess.myplugin' is missing required analysis configuration. Plugin must provide either 'analysis_config' attribute or 'get_analysis_config()' function. +``` + +Common configuration errors: +- **Missing configuration**: Plugin has no `analysis_config` or `get_analysis_config()` +- **Invalid format**: Configuration is not a dict or AnalysisConfig instance +- **Function errors**: `get_analysis_config()` raises an exception +- **Import errors**: Plugin module cannot be imported + +## Best Practices + +### 1. Choose Appropriate Comparison Keys +- Use version-related labels for tracking releases: `["version", "build"]` +- Include model/configuration changes: `["model", "parameters"]` +- Avoid high-cardinality labels that create too many baseline groups + +### 2. Set Realistic Thresholds +- Start with conservative thresholds (10-20%) and adjust based on your system's variability +- Consider different thresholds for different KPI types (latency vs throughput) +- Use stricter thresholds in production environments + +### 3. Ignore Environmental Noise +- Always ignore machine-specific labels: `["hostname", "worker_id"]` +- Ignore timing metadata: `["timestamp", "run_id", "job_id"]` +- Consider ignoring OS/infrastructure differences if testing across platforms + +### 4. Organize Results Clearly +- Use logical sorting keys that help humans understand results +- Group related configurations together: `["platform", "model", "version"]` +- Put most important differentiators first + +### 5. Require Sufficient Baselines +- Set `min_baseline_points` to 2-3 for better statistical reliability +- Consider higher values for critical production metrics +- Balance reliability with the need to run analysis on limited historical data + +## Integration with Orchestration + +The analysis configuration works with the orchestration layer: + +```yaml +# In your caliper postprocess config +caliper: + postprocess: + analyze: + enabled: true + fail_on_regression: false # Don't fail tests on regression by default + current_kpis: "kpis.json" + historical_kpis: "historical_data/" + output: "kpi_analysis.json" +``` + +When `analyze.enabled: true`, Caliper will: +1. Load your plugin's analysis configuration +2. Compare current KPIs against historical baselines +3. Generate a JSON analysis report +4. Optionally fail the test if regressions are detected (when `fail_on_regression: true`) From 9c9f82cca1b964dd77be2be1758cf4e50281d6c8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 15:44:52 +0200 Subject: [PATCH 0423/1072] [caliper] engine: kpi: format: include the y_unit --- projects/caliper/engine/kpi/format.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py index 009cb0e94..697cbe473 100644 --- a/projects/caliper/engine/kpi/format.py +++ b/projects/caliper/engine/kpi/format.py @@ -119,10 +119,13 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: kpi_record: dict[str, Any] = { "id": kpi_id, "value": final_value, - "unit": kpi.get("unit"), "higher_is_better": kpi_labels.get("higher_is_better", True), "is_2d": is_2d, } + + # Only add unit field for non-2D KPIs + if not is_2d: + kpi_record["unit"] = kpi.get("unit") if kpi_specific_labels: kpi_record["labels"] = kpi_specific_labels @@ -146,7 +149,8 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: { "x_unit": getattr(func, "_kpi_x_unit", ""), "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_unit": getattr(func, "_kpi_y_unit", None) + or getattr(func, "_kpi_unit", ""), "y_help": getattr(func, "_kpi_y_help", None) or getattr(func, "_kpi_help", ""), } @@ -164,6 +168,17 @@ def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: } ) + # Add 2D-specific fallback metadata if this is a 2D KPI + if is_2d: + kpi_record.update( + { + "x_unit": kpi.get("x_unit", ""), + "x_help": kpi.get("x_help", ""), + "y_unit": kpi.get("y_unit", kpi.get("unit", "")), + "y_help": kpi.get("y_help", ""), + } + ) + test_data["kpis"].append(kpi_record) # Convert to final structure From a49104f00b730f22c7635387327a2299b3c7c60c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 15:52:40 +0200 Subject: [PATCH 0424/1072] [guidellm] postprocess: guidellm: parsing/kpis: remove guidellm_input_tokens_per_request KPI --- .../guidellm/postprocess/guidellm/parsing/kpis.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index 35ecf5a85..cfa21ddfa 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -31,17 +31,6 @@ def guidellm_request_concurrency(unified_record) -> float: # Token Count Statistics KPIs - static values -@LowerBetter() -@Format("{:.1f}") -@KPIMetadata(help="Average input tokens per request", unit="tokens") -def guidellm_input_tokens_per_request(unified_record) -> float: - """Input Tokens Per Request KPI.""" - value = unified_record.metrics.get("input_tokens_per_request") - if value is None: - raise ValueError("input_tokens_per_request metric not found") - return float(value) - - @LowerBetter() @Format("{:.1f}") @KPIMetadata(help="Average output tokens per request", unit="tokens") From 9a1056d546f3cb5c97e60c17ebc5a15e81631d3c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 17:09:33 +0200 Subject: [PATCH 0425/1072] [projects] postprocess: remove the kpi_catalog --- projects/caliper/engine/kpi/catalog.py | 9 --------- projects/caliper/engine/model.py | 4 ---- projects/caliper/tests/stub_plugin.py | 10 ---------- projects/guidellm/postprocess/guidellm/plugin.py | 4 ---- .../mcp_gateway/postprocess/mcp_gateway/plugin.py | 3 --- .../postprocess/tests/test_mcp_gateway_plugin.py | 12 ------------ projects/rhaiis/postprocess/plugin.py | 3 --- projects/skeleton/postprocess/default/plugin.py | 4 ---- 8 files changed, 49 deletions(-) delete mode 100644 projects/caliper/engine/kpi/catalog.py diff --git a/projects/caliper/engine/kpi/catalog.py b/projects/caliper/engine/kpi/catalog.py deleted file mode 100644 index 55e41c389..000000000 --- a/projects/caliper/engine/kpi/catalog.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Resolve KPI catalog from plugin.""" - -from __future__ import annotations - -from projects.caliper.engine.model import PostProcessingPlugin - - -def get_catalog(plugin: PostProcessingPlugin) -> list[dict[str, object]]: - return plugin.kpi_catalog() diff --git a/projects/caliper/engine/model.py b/projects/caliper/engine/model.py index c04b4c845..0a11099b6 100644 --- a/projects/caliper/engine/model.py +++ b/projects/caliper/engine/model.py @@ -97,10 +97,6 @@ def visualize( """Write plots/HTML; return list of created file paths.""" return [] - def kpi_catalog(self) -> list[dict[str, Any]]: - """KPI definitions (id, name, unit, higher_is_better).""" - return [] - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Return canonical-shaped KPI dicts (pre-validated).""" return [] diff --git a/projects/caliper/tests/stub_plugin.py b/projects/caliper/tests/stub_plugin.py index 899347351..1283524a4 100644 --- a/projects/caliper/tests/stub_plugin.py +++ b/projects/caliper/tests/stub_plugin.py @@ -63,16 +63,6 @@ def visualize( ) return [str(html_path)] - def kpi_catalog(self) -> list[dict[str, object]]: - return [ - { - "kpi_id": "throughput_rps", - "name": "Throughput", - "unit": "req/s", - "higher_is_better": True, - } - ] - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, object]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, object]] = [] diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index d58a700ab..5ab512841 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -189,10 +189,6 @@ def visualize( return paths - def kpi_catalog(self) -> list[dict[str, Any]]: - """Return the GuideLLM KPI catalog.""" - return self.kpi_handler.get_catalog() - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Compute KPI values from the unified model.""" return self.kpi_handler.compute_kpis(model) diff --git a/projects/mcp_gateway/postprocess/mcp_gateway/plugin.py b/projects/mcp_gateway/postprocess/mcp_gateway/plugin.py index 5d9932e2b..78b5cdd50 100644 --- a/projects/mcp_gateway/postprocess/mcp_gateway/plugin.py +++ b/projects/mcp_gateway/postprocess/mcp_gateway/plugin.py @@ -27,9 +27,6 @@ def __init__(self): def parse(self, nodes: list[TestBaseNode]) -> ParseResult: return self.parser.parse(nodes) - def kpi_catalog(self) -> list[dict[str, Any]]: - return self.kpi_handler.get_catalog() - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: return self.kpi_handler.compute_kpis(model) diff --git a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py index e27a963fc..623498d9e 100644 --- a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py +++ b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py @@ -133,15 +133,6 @@ def test_parse_multiple_nodes(self, tmp_path: Path): class TestMCPGatewayKpis: - def test_catalog_has_expected_kpis(self): - handler = MCPGatewayKpiHandler() - catalog = handler.get_catalog() - kpi_ids = {k["kpi_id"] for k in catalog} - - assert "mcp_gw_requests_per_second" in kpi_ids - assert "mcp_gw_p95_ms" in kpi_ids - assert "mcp_gw_failure_rate" in kpi_ids - def test_compute_kpis(self, tmp_path: Path): node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) parser = MCPGatewayParser() @@ -210,9 +201,6 @@ def test_plugin_parse_and_kpis(self, tmp_path: Path): kpis = plugin.compute_kpis(model) assert len(kpis) > 0 - catalog = plugin.kpi_catalog() - assert len(catalog) > 0 - def test_visualize_returns_empty(self, tmp_path: Path): plugin = get_plugin() model = UnifiedRunModel( diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 3c6765053..7557c8f79 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -108,9 +108,6 @@ def visualize( ) -> list[str]: return [] - def kpi_catalog(self) -> list[dict[str, Any]]: - return self.kpi_handler.get_catalog() - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: return self.kpi_handler.compute_kpis(model) diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 18b00075f..d19a27f24 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -88,10 +88,6 @@ def visualize( ) return paths - def kpi_catalog(self) -> list[dict[str, Any]]: - """Return the KPI catalog.""" - return self.kpi_handler.get_catalog() - def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Compute KPI values from the unified model.""" return self.kpi_handler.compute_kpis(model) From 140e8504bd41aa3ff5be27435fcb1c05f108446b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 19:11:44 +0200 Subject: [PATCH 0426/1072] docs: caliper: plugin_kpis: new doc --- docs/caliper/plugin_kpis.md | 391 ++++++++++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 docs/caliper/plugin_kpis.md diff --git a/docs/caliper/plugin_kpis.md b/docs/caliper/plugin_kpis.md new file mode 100644 index 000000000..28dcbaecc --- /dev/null +++ b/docs/caliper/plugin_kpis.md @@ -0,0 +1,391 @@ +# Plugin KPI Development Guide + +This document explains how to build and expose KPIs (Key Performance Indicators) in Caliper plugins. + +## Overview + +Caliper plugins define KPIs as decorated Python functions that extract performance metrics from test results. The KPI system supports both scalar metrics and 2D performance curves, with rich metadata for visualization and analysis. + +## KPI Function Structure + +### Basic KPI Function + +```python +from projects.caliper.engine.kpi import KPIMetadata, HigherBetter + + +@HigherBetter() +@KPIMetadata(help="Request throughput in requests per second", unit="req/s") +def request_rate(unified_record) -> float: + """Request Rate KPI.""" + value = unified_record.metrics.get("request_rate") + if value is None: + raise ValueError("request_rate metric not found") + return float(value) +``` + +### Function Requirements + +1. **Function name**: Becomes the KPI ID in output +2. **Single parameter**: `unified_record` - the parsed test result +3. **Return type**: `float` for scalar KPIs, `list[tuple[float, float]]` for 2D KPIs +4. **Docstring**: First line becomes the display name (without " KPI.") +5. **Decorators**: Required for metadata and behavior + +## KPI Decorators + +### Required Decorators + +**`@KPIMetadata(help, unit)`** +- `help`: Human-readable description of the metric +- `unit`: Unit of measurement (e.g., "req/s", "ms", "tokens") + +**Comparison Direction** +- `@HigherBetter()`: Higher values indicate better performance +- `@LowerBetter()`: Lower values indicate better performance + +### Optional Decorators + +**`@Format(format_str)`** +- Specify number formatting (e.g., `"{:.1f}"`, `"{:.2%}"`) + +**`@TwoDimensional(x_unit, x_help, y_unit=None, y_help=None)`** +- Marks KPI as returning 2D data (performance curves) +- `x_unit`/`x_help`: X-axis unit and description +- `y_unit`/`y_help`: Y-axis unit and description (defaults to main unit/help) + +## 2D KPIs (Performance Curves) + +2D KPIs return lists of (x, y) coordinate pairs representing performance curves: + +```python +@HigherBetter() +@TwoDimensional( + x_unit="req/s", x_help="Request rate", y_unit="tokens/s", y_help="Achieved throughput" +) +@KPIMetadata(help="Throughput achieved at different request rates", unit="tokens/s") +def throughput_curve(unified_record) -> list[tuple[float, float]]: + """Throughput vs Request Rate Curve KPI.""" + request_rates = unified_record.metrics.get("request_rate", []) + throughputs = unified_record.metrics.get("throughput", []) + + return [(float(x), float(y)) for x, y in zip(request_rates, throughputs)] +``` + +### 2D KPI Output Format + +2D KPIs generate structured JSON output: + +```json +{ + "id": "throughput_curve", + "value": { + "data_points": [ + {"x": 1.0, "y": 150.2}, + {"x": 2.0, "y": 298.5} + ], + "count": 2 + }, + "higher_is_better": true, + "is_2d": true, + "name": "Throughput vs Request Rate Curve", + "help": "Throughput achieved at different request rates", + "x_unit": "req/s", + "x_help": "Request rate", + "y_unit": "tokens/s", + "y_help": "Achieved throughput" +} +``` + +## Plugin Integration + +### KPI Function Discovery + +KPI functions are discovered through two mechanisms: + +1. **During KPI Computation**: The KPI handler uses `get_kpi_functions(inspect.getmodule(KpiHandler))` to find functions in the handler's module +2. **During Format Transformation**: The hierarchical format transformer attempts to import the plugin module for metadata + +```python +# In KPI handler module (e.g., projects/myplugin/postprocess/myplugin/parsing/kpis.py) +from projects.caliper.engine.kpi import KPIMetadata, HigherBetter + + +@HigherBetter() +@KPIMetadata(help="My metric", unit="units") +def my_kpi_function(unified_record) -> float: + """My KPI.""" + return 42.0 +``` + +### Plugin Module Structure + +**For KPI Computation**: Functions are discovered in the KPI handler's module using `inspect.getmodule()`. + +**For Metadata Extraction**: The hierarchical format transformer attempts to import the main plugin module to extract decorator metadata. If this fails (e.g., due to heavy dependencies), the transformer preserves metadata from the original v1 KPI records, ensuring 2D KPI information (`x_unit`, `y_unit`, `x_help`, `y_help`) is not lost during format conversion. + +### Integration with PostProcessingPlugin + +Your plugin class should implement the `compute_kpis` method to generate KPI records: + +```python +class MyPlugin(PostProcessingPlugin): + def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: + """Compute KPI values from the unified model.""" + return self.kpi_handler.compute_kpis(model) +``` + +## KPI Handler Pattern + +### Standard KPI Handler Implementation + +```python +import inspect +from projects.caliper.engine.kpi.decorators import get_kpi_functions + + +class MyKpiHandler: + def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: + """Generate KPI records from unified model.""" + kpi_functions = get_kpi_functions(inspect.getmodule(MyKpiHandler)) + records = [] + + for record in model.unified_result_records: + for kpi_name, kpi_func in kpi_functions.items(): + try: + value = kpi_func(record) + + # Base KPI record + kpi_record = { + "schema_version": "1", + "kpi_id": kpi_name, + "value": value, + "unit": kpi_func._kpi_unit, + "run_id": record.test_base_path, + "timestamp": "2024-01-01T00:00:00Z", + "labels": { + "higher_is_better": kpi_func._kpi_higher_is_better, + # Add other labels from record + }, + "source": {"test_base_path": record.test_base_path}, + } + + # Add 2D-specific metadata if this is a 2D KPI + if getattr(kpi_func, "_kpi_is_2d", False): + kpi_record.update( + { + "is_2d": True, + "x_unit": kpi_func._kpi_x_unit, + "x_help": kpi_func._kpi_x_help, + "y_unit": getattr(kpi_func, "_kpi_y_unit", None) + or kpi_func._kpi_unit, + "y_help": getattr(kpi_func, "_kpi_y_help", None) + or kpi_func._kpi_help, + } + ) + + records.append(kpi_record) + except Exception as e: + # Handle missing metrics gracefully + continue + + return records +``` + +## Best Practices + +### 1. Error Handling + +```python +@HigherBetter() +@KPIMetadata(help="Robust metric", unit="units") +def robust_kpi(unified_record) -> float: + """Robust KPI.""" + value = unified_record.metrics.get("my_metric") + if value is None: + raise ValueError("my_metric not found in record") + + # Validate data type + if not isinstance(value, (int, float)): + raise ValueError(f"Expected numeric value, got {type(value)}") + + return float(value) +``` + +### 2. Label Extraction + +```python +def extract_test_labels(record) -> dict[str, Any]: + """Extract labels for KPI records.""" + return { + "platform": record.distinguishing_labels.get("platform", "unknown"), + "model": record.metrics.get("model_name", "unknown"), + "version": record.metrics.get("product_version", "unknown"), + } +``` + +### 3. 2D Data Validation + +```python +@TwoDimensional(x_unit="x", x_help="X values", y_unit="y", y_help="Y values") +@KPIMetadata(help="Performance curve", unit="y") +def performance_curve(unified_record) -> list[tuple[float, float]]: + """Performance Curve KPI.""" + x_values = unified_record.metrics.get("x_data", []) + y_values = unified_record.metrics.get("y_data", []) + + if len(x_values) != len(y_values): + raise ValueError("X and Y data arrays must have same length") + + if not x_values: + return [] # Return empty list for missing data + + return [(float(x), float(y)) for x, y in zip(x_values, y_values)] +``` + +### 4. Conditional KPIs + +```python +@HigherBetter() +@KPIMetadata(help="Optional metric", unit="units") +def optional_kpi(unified_record) -> float: + """Optional KPI.""" + # Only compute for certain test types + if not unified_record.metrics.get("enable_optional_metrics", False): + raise ValueError("Optional metrics disabled for this test") + + return unified_record.metrics.get("optional_value", 0.0) +``` + +### 5. 2D Metadata Preservation + +Always include 2D metadata in your KPI handler implementation to ensure proper display: + +```python +# In your KPI handler's compute_kpis method +if getattr(kpi_func, "_kpi_is_2d", False): + kpi_record.update( + { + "is_2d": True, + "x_unit": kpi_func._kpi_x_unit, + "x_help": kpi_func._kpi_x_help, + "y_unit": getattr(kpi_func, "_kpi_y_unit", None) or kpi_func._kpi_unit, + "y_help": getattr(kpi_func, "_kpi_y_help", None) or kpi_func._kpi_help, + } + ) +``` + +This metadata is preserved during v1→v2 format transformation even if the plugin module cannot be imported due to dependencies. + +## Output Formats + +### Hierarchical JSON (Schema v2) + +The default output format groups KPIs by test with metadata: + +```json +{ + "schema_version": "2", + "tests": [ + { + "run_id": "test_001", + "labels": {"platform": "gpu", "model": "llama"}, + "metadata": {"timestamp": "2024-01-01T00:00:00Z"}, + "kpis": [ + { + "id": "throughput", + "value": 150.5, + "higher_is_better": true, + "is_2d": false, + "unit": "tokens/s", + "name": "Throughput", + "help": "Token generation rate" + } + ] + } + ] +} +``` + +### JSONL Format (Schema v1) + +Legacy flat format with one KPI per line: + +```json +{"kpi_id": "throughput", "value": 150.5, "unit": "tokens/s", "run_id": "test_001", ...} +``` + +## Testing KPIs + +### Unit Testing KPI Functions + +```python +import pytest +from unittest.mock import Mock + + +def test_throughput_kpi(): + # Create mock record + mock_record = Mock() + mock_record.metrics = {"throughput": 150.5} + + # Test KPI function + result = throughput_kpi(mock_record) + + assert result == 150.5 + assert isinstance(result, float) + + +def test_throughput_kpi_missing_data(): + mock_record = Mock() + mock_record.metrics = {} + + with pytest.raises(ValueError, match="throughput metric not found"): + throughput_kpi(mock_record) +``` + +### Integration Testing + +```python +def test_kpi_generation(sample_unified_model): + handler = MyKpiHandler() + kpis = handler.compute_kpis(sample_unified_model) + + assert len(kpis) > 0 + assert all("kpi_id" in kpi for kpi in kpis) + assert all("value" in kpi for kpi in kpis) +``` + +## Common Patterns + +### Aggregation KPIs + +```python +@HigherBetter() +@KPIMetadata(help="Average performance across tests", unit="req/s") +def average_performance(unified_record) -> float: + """Average Performance KPI.""" + values = unified_record.metrics.get("performance_samples", []) + if not values: + raise ValueError("No performance samples found") + + return sum(values) / len(values) +``` + +### Derived Metrics + +```python +@LowerBetter() +@KPIMetadata(help="Efficiency ratio", unit="ratio") +def efficiency_ratio(unified_record) -> float: + """Efficiency Ratio KPI.""" + throughput = unified_record.metrics.get("throughput") + latency = unified_record.metrics.get("latency") + + if not throughput or not latency: + raise ValueError("Both throughput and latency required") + + return throughput / latency +``` + +This guide provides the foundation for implementing robust, well-documented KPIs in Caliper plugins. From 0934c328d2975e7f262808df80b1981208db5440 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 19:29:14 +0200 Subject: [PATCH 0427/1072] [caliper] tests: fix --- projects/caliper/tests/stub_plugin.py | 5 +++++ projects/caliper/tests/test_kpi_analyze.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/projects/caliper/tests/stub_plugin.py b/projects/caliper/tests/stub_plugin.py index 1283524a4..2d9ac5313 100644 --- a/projects/caliper/tests/stub_plugin.py +++ b/projects/caliper/tests/stub_plugin.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime from pathlib import Path +from projects.caliper.engine.kpi.analyze import AnalysisConfig from projects.caliper.engine.model import ( ParseResult, PostProcessingPlugin, @@ -101,5 +102,9 @@ def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, object]: } +# Analysis configuration for testing KPI regression analysis +analysis_config = AnalysisConfig() + + def get_plugin() -> PostProcessingPlugin: return StubPlugin() diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py index 2ca06e4de..456510433 100644 --- a/projects/caliper/tests/test_kpi_analyze.py +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -201,7 +201,7 @@ def test_pass_no_regression(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) assert test_status["exit_code"] == 0 @@ -251,7 +251,7 @@ def test_regression_detected(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) assert test_status["exit_code"] == 3 @@ -286,7 +286,7 @@ def test_no_historical_data(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) assert test_status["exit_code"] == 2 @@ -331,7 +331,7 @@ def test_mixed_regression_and_pass(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) assert test_status["exit_code"] == 3 @@ -380,7 +380,7 @@ def test_skips_non_scalar_kpis(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) assert test_status["exit_code"] == 0 @@ -430,7 +430,7 @@ def test_comparison_keys_separate_baselines(self, tmp_path): current_kpi_file=current_dir / "kpis.json", historical_data_dir=historical_dir, output_file=output_file, - plugin_module="nonexistent_plugin", + plugin_module="projects.caliper.tests.stub_plugin", ) # Without plugin config, default AnalysisConfig has no comparison_keys, From 529b0ad0f5de0be0e00d3c7f4566fbf3692695c8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 19:43:44 +0200 Subject: [PATCH 0428/1072] [guidellm] postprocess: guidellm: use the input concurrency level --- .../postprocess/guidellm/parsing/kpis.py | 124 +++++++++--------- .../postprocess/guidellm/parsing/models.py | 4 +- .../postprocess/guidellm/parsing/parsers.py | 13 +- .../guidellm/tests/test_postprocess_parser.py | 12 +- 4 files changed, 84 insertions(+), 69 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index cfa21ddfa..2c0172c88 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -51,156 +51,160 @@ def guidellm_output_tokens_per_request(unified_record) -> float: # 2D KPIs that extract data from performance curves @HigherBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="tokens/s", y_help="Achieved throughput", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.1f}", ) -@KPIMetadata(help="Throughput achieved at different request rates", unit="tokens/s") -def guidellm_throughput_curve(unified_record) -> list[tuple[float, float]]: - """Throughput vs Request Rate Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="Throughput achieved at different concurrency levels", unit="tokens/s") +def guidellm_throughput_curve(unified_record) -> list[tuple[int, float]]: + """Throughput vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) tokens_per_sec = curves.get("tokens_per_second", []) - if not request_rates or not tokens_per_sec or len(request_rates) != len(tokens_per_sec): + if ( + not intended_concurrency + or not tokens_per_sec + or len(intended_concurrency) != len(tokens_per_sec) + ): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, tokens_per_sec, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, tokens_per_sec, strict=False) if x > 0 and y > 0 ] @LowerBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="s", y_help="P95 latency", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.4f}", ) -@KPIMetadata(help="P95 latency at different request rates", unit="s") -def guidellm_latency(unified_record) -> list[tuple[float, float]]: - """P95 Latency vs Load Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="P95 latency at different concurrency levels", unit="s") +def guidellm_latency(unified_record) -> list[tuple[int, float]]: + """P95 Latency vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) p95_latency = curves.get("request_latency_p95", []) - if not request_rates or not p95_latency or len(request_rates) != len(p95_latency): + if not intended_concurrency or not p95_latency or len(intended_concurrency) != len(p95_latency): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, p95_latency, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, p95_latency, strict=False) if x > 0 and y > 0 ] @LowerBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="s", y_help="TTFT P95", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.4f}", ) -@KPIMetadata(help="Time to first token P95 at different request rates", unit="s") -def guidellm_ttft(unified_record) -> list[tuple[float, float]]: - """TTFT P95 vs Load Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="Time to first token P95 at different concurrency levels", unit="s") +def guidellm_ttft(unified_record) -> list[tuple[int, float]]: + """TTFT P95 vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) ttft_p95 = curves.get("ttft_p95", []) - if not request_rates or not ttft_p95 or len(request_rates) != len(ttft_p95): + if not intended_concurrency or not ttft_p95 or len(intended_concurrency) != len(ttft_p95): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, ttft_p95, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, ttft_p95, strict=False) if x > 0 and y > 0 ] @LowerBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="s", y_help="TPOT median", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.4f}", ) -@KPIMetadata(help="Time per output token median at different request rates", unit="s") -def guidellm_tpot(unified_record) -> list[tuple[float, float]]: - """TPOT Median vs Load Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="Time per output token median at different concurrency levels", unit="s") +def guidellm_tpot(unified_record) -> list[tuple[int, float]]: + """TPOT Median vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) tpot_median = curves.get("tpot_median", []) - if not request_rates or not tpot_median or len(request_rates) != len(tpot_median): + if not intended_concurrency or not tpot_median or len(intended_concurrency) != len(tpot_median): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, tpot_median, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, tpot_median, strict=False) if x > 0 and y > 0 ] @LowerBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="s", y_help="TPOT P95", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.4f}", ) -@KPIMetadata(help="Time per output token P95 at different request rates", unit="s") -def guidellm_tpot_p95(unified_record) -> list[tuple[float, float]]: - """TPOT P95 vs Load Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="Time per output token P95 at different concurrency levels", unit="s") +def guidellm_tpot_p95(unified_record) -> list[tuple[int, float]]: + """TPOT P95 vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) tpot_p95 = curves.get("tpot_p95", []) - if not request_rates or not tpot_p95 or len(request_rates) != len(tpot_p95): + if not intended_concurrency or not tpot_p95 or len(intended_concurrency) != len(tpot_p95): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, tpot_p95, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, tpot_p95, strict=False) if x > 0 and y > 0 ] @LowerBetter() @TwoDimensional( - x_unit="req/s", - x_help="Request rate", + x_unit="connections", + x_help="Input concurrency", y_unit="s", y_help="TPOT P99", - x_format="{:.1f}", + x_format="{:.0f}", y_format="{:.4f}", ) -@KPIMetadata(help="Time per output token P99 at different request rates", unit="s") -def guidellm_tpot_p99(unified_record) -> list[tuple[float, float]]: - """TPOT P99 vs Load Curve KPI.""" - request_rates = unified_record.metrics.get("request_rate", []) +@KPIMetadata(help="Time per output token P99 at different concurrency levels", unit="s") +def guidellm_tpot_p99(unified_record) -> list[tuple[int, float]]: + """TPOT P99 vs Concurrency Curve KPI.""" curves = unified_record.metrics.get("performance_curves", {}) + intended_concurrency = curves.get("intended_concurrency", []) tpot_p99 = curves.get("tpot_p99", []) - if not request_rates or not tpot_p99 or len(request_rates) != len(tpot_p99): + if not intended_concurrency or not tpot_p99 or len(intended_concurrency) != len(tpot_p99): return [] return [ - (float(x), float(y)) - for x, y in zip(request_rates, tpot_p99, strict=False) + (int(x), float(y)) + for x, y in zip(intended_concurrency, tpot_p99, strict=False) if x > 0 and y > 0 ] diff --git a/projects/guidellm/postprocess/guidellm/parsing/models.py b/projects/guidellm/postprocess/guidellm/parsing/models.py index 7bf4d06a6..fa338cf9a 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/models.py +++ b/projects/guidellm/postprocess/guidellm/parsing/models.py @@ -17,7 +17,8 @@ class GuideLLMBenchmark: # Request metrics request_rate: float = 0.0 - request_concurrency: float = 0.0 + request_concurrency: float = 0.0 # Effective concurrency achieved + intended_concurrency: int = 0 # Input/target concurrency completed_requests: int = 0 failed_requests: int = 0 @@ -72,6 +73,7 @@ def to_dict(self) -> dict[str, Any]: "duration": self.duration, "request_rate": self.request_rate, "request_concurrency": self.request_concurrency, + "intended_concurrency": self.intended_concurrency, "completed_requests": self.completed_requests, "failed_requests": self.failed_requests, "input_tokens_per_request": self.input_tokens_per_request, diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index b6674e75e..bee45920b 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -257,7 +257,7 @@ def _parse_single_benchmark(self, benchmark_data: dict[str, Any]) -> GuideLLMBen strategy = strategy_info.get("type_", "unknown") # Extract concurrency (streams) with multiple fallback paths - concurrency = self._extract_concurrency(strategy_info, scheduler) + intended_concurrency = self._extract_concurrency(strategy_info, scheduler) # Extract timing info state = scheduler.get("state", {}) @@ -343,7 +343,8 @@ def get_metric_value( cooldown_time=0.0, # Not available in JSON format # Request metrics request_rate=request_rate, - request_concurrency=concurrency, + request_concurrency=intended_concurrency, # For now, effective = intended + intended_concurrency=int(intended_concurrency), completed_requests=completed_requests, failed_requests=0, # Could extract from unsuccessful metrics if needed # Token metrics per request @@ -485,7 +486,8 @@ def _create_aggregated_metrics( "request_latency_p95": [], "completed_requests": [], "failed_requests": [], - "request_concurrency": [], # Add concurrency to curves + "request_concurrency": [], # Effective concurrency to curves + "intended_concurrency": [], # Intended concurrency to curves } for benchmark in sorted_benchmarks: @@ -508,7 +510,10 @@ def _create_aggregated_metrics( curves["failed_requests"].append(benchmark.failed_requests) curves["request_concurrency"].append( benchmark.request_concurrency - ) # Add concurrency per rate point + ) # Add effective concurrency per rate point + curves["intended_concurrency"].append( + benchmark.intended_concurrency + ) # Add intended concurrency per rate point # Add request_rate and performance curves to metrics metrics["request_rate"] = request_rates diff --git a/projects/guidellm/tests/test_postprocess_parser.py b/projects/guidellm/tests/test_postprocess_parser.py index caa357f9a..4d66a0193 100644 --- a/projects/guidellm/tests/test_postprocess_parser.py +++ b/projects/guidellm/tests/test_postprocess_parser.py @@ -73,9 +73,13 @@ def test_parser_accepts_rate_split_benchmark_files(tmp_path: Path) -> None: artifact_paths=[file_b, file_a], ) - result = parser.parse(tmp_path, [node]) + result = parser.parse([node]) assert result.warnings == [] - assert len(result.records) == 2 - assert [record.metrics["request_concurrency"] for record in result.records] == [32.0, 64.0] - assert all(record.run_identity == {"guidellm": True} for record in result.records) + assert len(result.records) == 1 + + # Check that both rate configurations are present in the performance curves + record = result.records[0] + curves = record.metrics["performance_curves"] + assert curves["intended_concurrency"] == [32, 64] + assert record.run_identity == {"guidellm": True} From 174c77db2caa69aa769d949ea4f2558c51bfed66 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 4 Aug 2026 22:44:17 +0200 Subject: [PATCH 0429/1072] [caliper] orchestration: cli_builder: build_s3_export_command: include --upload-id and --dry-run flags --- projects/caliper/orchestration/cli_builder.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 8342da7e7..58daa50cd 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -452,6 +452,13 @@ def build_s3_export_command( if analysis_file: cmd.extend(["--analysis-file", str(analysis_file)]) + # S3 export options + if s3_config.export.upload_id: + cmd.extend(["--upload-id", s3_config.export.upload_id]) + + if s3_config.export.dry_run: + cmd.append("--dry-run") + # Vault configuration cmd.extend(["--vault", s3_config.vault.name]) cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) From 81ac1542e8f15bbfeec0b3793bc02c6379ceb6ac Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 12:51:13 +0300 Subject: [PATCH 0430/1072] Add 2D metric support to MLflow export pipeline KPIs with is_2d=true (e.g. throughput curves, latency vs load) are now extracted as {x,y} data points in metrics.json and logged as stepped MLflow metrics so the UI renders them as curves. Co-authored-by: Cursor --- .../engine/file_export/mlflow_backend.py | 33 +++++++++++++++++-- .../caliper/engine/kpi/metrics_from_kpis.py | 32 ++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/projects/caliper/engine/file_export/mlflow_backend.py b/projects/caliper/engine/file_export/mlflow_backend.py index 2d199e34c..959af11c1 100644 --- a/projects/caliper/engine/file_export/mlflow_backend.py +++ b/projects/caliper/engine/file_export/mlflow_backend.py @@ -461,6 +461,31 @@ def _load_json_file(path: Path) -> dict[str, Any]: return {} +def _log_2d_metrics(metrics_2d: dict[str, Any]) -> None: + """Log 2D metrics as stepped MLflow metrics. + + Each key maps to a list of ``{"x": ..., "y": ...}`` dicts (already + sorted by x in ``metrics_from_kpis``). Each data point is logged + with ``step=int(x)`` so MLflow renders the curve. + """ + import mlflow + + for metric_name, data_points in metrics_2d.items(): + if not isinstance(data_points, list): + continue + for pt in data_points: + if isinstance(pt, dict): + x = pt.get("x") + y = pt.get("y") + if ( + isinstance(x, int | float) + and not isinstance(x, bool) + and isinstance(y, int | float) + and not isinstance(y, bool) + ): + mlflow.log_metric(str(metric_name), float(y), step=int(x)) + + def _log_metrics_and_params_from_tree(artifact_root: Path) -> None: """Find metrics.json/parameters.json under __test_labels__.yaml-marked dirs and log them.""" import mlflow @@ -473,7 +498,9 @@ def _log_metrics_and_params_from_tree(artifact_root: Path) -> None: mf = run_dir / "metrics.json" if mf.is_file(): for k, v in _load_json_file(mf).items(): - if isinstance(v, int | float) and not isinstance(v, bool): + if isinstance(v, list): + _log_2d_metrics({k: v}) + elif isinstance(v, int | float) and not isinstance(v, bool): mlflow.log_metric(str(k), float(v)) pf = run_dir / "parameters.json" @@ -593,7 +620,9 @@ def _run(uri: str | None) -> tuple[str, dict[str, Any] | None]: mf = run_dir / metrics_file if mf.is_file(): for k, v in _load_json_file(mf).items(): - if isinstance(v, int | float) and not isinstance(v, bool): + if isinstance(v, list): + _log_2d_metrics({k: v}) + elif isinstance(v, int | float) and not isinstance(v, bool): mlflow.log_metric(str(k), float(v)) pf = run_dir / parameters_file diff --git a/projects/caliper/engine/kpi/metrics_from_kpis.py b/projects/caliper/engine/kpi/metrics_from_kpis.py index 408ae4981..77848bff0 100644 --- a/projects/caliper/engine/kpi/metrics_from_kpis.py +++ b/projects/caliper/engine/kpi/metrics_from_kpis.py @@ -44,6 +44,27 @@ def _is_scalar(value: Any) -> bool: return isinstance(value, int | float) and not isinstance(value, bool) +def _extract_2d_points(value: Any) -> list[dict[str, float]] | None: + """Extract sorted (x, y) data points from a 2D KPI value. + + Returns a list of ``{"x": ..., "y": ...}`` dicts sorted by x, + or ``None`` if the value is not a valid 2D structure. + """ + if not isinstance(value, dict): + return None + data_points = value.get("data_points") + if not isinstance(data_points, list) or not data_points: + return None + points = [] + for pt in data_points: + if isinstance(pt, dict) and _is_scalar(pt.get("x")) and _is_scalar(pt.get("y")): + points.append({"x": float(pt["x"]), "y": float(pt["y"])}) + if not points: + return None + points.sort(key=lambda p: p["x"]) + return points + + def _write_json(path: Path, data: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as f: @@ -108,11 +129,18 @@ def generate_metrics_from_kpis( continue kpis = test_entry.get("kpis", []) - metrics: dict[str, float] = {} + metrics: dict[str, Any] = {} for kpi in kpis: kpi_id = kpi.get("id", "") + if not kpi_id: + continue value = kpi.get("value") - if kpi_id and _is_scalar(value): + is_2d = kpi.get("is_2d", False) + if is_2d: + points = _extract_2d_points(value) + if points: + metrics[kpi_id] = points + elif _is_scalar(value): metrics[kpi_id] = value if metrics: From 67ccb4b0727d10dbcf9fe0de9528917326412031 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 13:59:16 +0300 Subject: [PATCH 0431/1072] Fix replot keep flag: config override was always ignored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The click is_flag=True default (False) prevented the config fallback caliper.replot.keep from ever being consulted, so /var overrides had no effect and the download directory was always cleaned up — deleting the metrics.json files before the export step could read them. Co-authored-by: Cursor --- projects/core/library/replot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index 81696785b..86699f428 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -159,7 +159,7 @@ def caliper_replot_entrypoint( if replot_url is not None: config.project.set_config("caliper.replot.url", replot_url) - status = run_replot(artifact_directory=artifact_directory, keep_download_dir=keep_download) + status = run_replot(artifact_directory=artifact_directory, keep_download_dir=keep_download or None) # Log the status import yaml From bcc8c9fa9096e0f144ac1ded33aaf09fad3c3aaa Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 14:30:43 +0300 Subject: [PATCH 0432/1072] Fix ruff format and validate 2D metric step values - Fix line length in replot.py for ruff format compliance. - Raise ValueError in _log_2d_metrics when x values are non-integer or x/y are non-numeric, instead of silently skipping. Co-authored-by: Cursor --- .../engine/file_export/mlflow_backend.py | 39 +++++++++++++------ projects/core/library/replot.py | 4 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/projects/caliper/engine/file_export/mlflow_backend.py b/projects/caliper/engine/file_export/mlflow_backend.py index 959af11c1..b3edc9ed4 100644 --- a/projects/caliper/engine/file_export/mlflow_backend.py +++ b/projects/caliper/engine/file_export/mlflow_backend.py @@ -467,23 +467,40 @@ def _log_2d_metrics(metrics_2d: dict[str, Any]) -> None: Each key maps to a list of ``{"x": ..., "y": ...}`` dicts (already sorted by x in ``metrics_from_kpis``). Each data point is logged with ``step=int(x)`` so MLflow renders the curve. + + Raises: + ValueError: If a data point has a non-integer x value (MLflow + steps must be integers) or if x/y are not numeric. """ import mlflow for metric_name, data_points in metrics_2d.items(): if not isinstance(data_points, list): continue - for pt in data_points: - if isinstance(pt, dict): - x = pt.get("x") - y = pt.get("y") - if ( - isinstance(x, int | float) - and not isinstance(x, bool) - and isinstance(y, int | float) - and not isinstance(y, bool) - ): - mlflow.log_metric(str(metric_name), float(y), step=int(x)) + for i, pt in enumerate(data_points): + if not isinstance(pt, dict): + raise ValueError( + f"2D metric {metric_name!r}: data_points[{i}] is {type(pt).__name__}, " + f"expected dict with 'x' and 'y' keys" + ) + x = pt.get("x") + y = pt.get("y") + if ( + not isinstance(x, int | float) + or isinstance(x, bool) + or not isinstance(y, int | float) + or isinstance(y, bool) + ): + raise ValueError( + f"2D metric {metric_name!r}: data_points[{i}] has non-numeric " + f"x={x!r} or y={y!r}" + ) + if x != int(x): + raise ValueError( + f"2D metric {metric_name!r}: data_points[{i}] has non-integer " + f"step x={x!r} (MLflow steps must be integers)" + ) + mlflow.log_metric(str(metric_name), float(y), step=int(x)) def _log_metrics_and_params_from_tree(artifact_root: Path) -> None: diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index 86699f428..2ca05938f 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -159,7 +159,9 @@ def caliper_replot_entrypoint( if replot_url is not None: config.project.set_config("caliper.replot.url", replot_url) - status = run_replot(artifact_directory=artifact_directory, keep_download_dir=keep_download or None) + status = run_replot( + artifact_directory=artifact_directory, keep_download_dir=keep_download or None + ) # Log the status import yaml From 567aa7e813751bcb500575ecee2042b07ffb84f9 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 17:13:56 +0300 Subject: [PATCH 0433/1072] Use default=None for --keep-download flag Cleaner than the or-None workaround: None default means "not passed" so the config fallback is used, while --keep-download / --no-keep-download explicitly override it. Co-authored-by: Cursor --- projects/core/library/replot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index 2ca05938f..553eeccbd 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -146,12 +146,13 @@ def run_replot(*, artifact_directory: Path | None, keep_download_dir: bool | Non @click.option( "--keep-download", is_flag=True, + default=None, help="Keep the download directory after processing (don't clean up).", ) @click.pass_context @ci_lib.safe_ci_entrypoint def caliper_replot_entrypoint( - _ctx, artifact_directory: Path | None, replot_url: str | None, keep_download: bool + _ctx, artifact_directory: Path | None, replot_url: str | None, keep_download: bool | None ): """Replot artifacts and visualizations from a remote URL.""" @@ -160,7 +161,7 @@ def caliper_replot_entrypoint( config.project.set_config("caliper.replot.url", replot_url) status = run_replot( - artifact_directory=artifact_directory, keep_download_dir=keep_download or None + artifact_directory=artifact_directory, keep_download_dir=keep_download ) # Log the status From 85160e1a103c5e10f3987dad5ee14a6a5a038e37 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 17:30:26 +0300 Subject: [PATCH 0434/1072] Fix ruff format: collapse single-line call Co-authored-by: Cursor --- projects/core/library/replot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index 553eeccbd..781ff73f6 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -160,9 +160,7 @@ def caliper_replot_entrypoint( if replot_url is not None: config.project.set_config("caliper.replot.url", replot_url) - status = run_replot( - artifact_directory=artifact_directory, keep_download_dir=keep_download - ) + status = run_replot(artifact_directory=artifact_directory, keep_download_dir=keep_download) # Log the status import yaml From 53136f1c967bc976763ec0e25574b179e26bf0ca Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 5 Aug 2026 10:39:54 -0400 Subject: [PATCH 0435/1072] refactor: use mlflow_destination in test labels instead of marker file - Extract MLflow pre-creation to _precreate_mlflow_run_if_configured() with guard pattern for readability - Move mlflow IDs out of labels into a separate mlflow_destination section in __test_labels__.yaml (run_id, experiment_id, workspace) - Remove __mlflow_precreated_run__.yaml marker file; export step and URL builder now read from mlflow_destination in test labels - Add conflict warning when export config and test labels both have differing run_ids - Update test-labels-format.md with mlflow_destination docs Co-authored-by: Cursor --- docs/caliper/test-labels-format.md | 8 ++ projects/caliper/orchestration/export.py | 99 ++++++++--------- projects/core/library/postprocess.py | 5 + projects/rhaiis/orchestration/test_phase.py | 111 +++++++++++--------- projects/rhaiis/postprocess/plugin.py | 30 +++++- 5 files changed, 147 insertions(+), 106 deletions(-) diff --git a/docs/caliper/test-labels-format.md b/docs/caliper/test-labels-format.md index bc2fbec3c..7f5316807 100644 --- a/docs/caliper/test-labels-format.md +++ b/docs/caliper/test-labels-format.md @@ -32,6 +32,10 @@ completion: - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`) - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`) - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`) +- **`mlflow_destination`**: Pre-created MLflow run for artifact upload + - **`run_id`**: MLflow run ID (assigned by the server during pre-creation) + - **`experiment_id`**: MLflow experiment ID + - **`workspace`**: MLflow workspace name (optional) - **`completion`**: Test execution status - **`success`**: `true` if succeeded, `false` if failed - **`message`**: Human-readable status description @@ -89,6 +93,10 @@ kpi_labels: platform: "CKS" gpu_type: "H100" test_harness: "guidellm" +mlflow_destination: + run_id: "48e49dfc966c487cb76cf105a5314908" + experiment_id: "264" + workspace: "forge-rhaiis" completion: success: true message: "Test completed successfully" diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index ac1cb0b5d..f675f2b7c 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -206,8 +206,19 @@ def run_from_orchestration_config( run_dirs = discover_run_dirs(from_path) - # Resume a pre-created MLflow run if the test step left a marker or test labels - mlflow_run_id = export_cfg.mlflow_run_id or _discover_precreated_mlflow_run_id(from_path) + # Resume a pre-created MLflow run if the test step left mlflow_destination in test labels + discovered_run_id = _discover_precreated_mlflow_run_id(from_path) + if ( + export_cfg.mlflow_run_id + and discovered_run_id + and export_cfg.mlflow_run_id != discovered_run_id + ): + logger.error( + "Conflicting MLflow run_ids: export config has %s, test labels have %s. Using export config.", + export_cfg.mlflow_run_id, + discovered_run_id, + ) + mlflow_run_id = export_cfg.mlflow_run_id or discovered_run_id # Resolve descriptive run names from labels + run_naming config naming = resolve_run_names( @@ -266,20 +277,7 @@ def run_from_orchestration_config( return yaml.safe_load(f.read()) -MLFLOW_PRECREATED_RUN_MARKER = "__mlflow_precreated_run__.yaml" -"""Marker file written by the test step to communicate the pre-created MLflow -run ID and experiment ID to the export step. - -The test step pre-creates an MLflow run (immediately ended) so that the -``run_id`` and ``experiment_id`` are known *before* CSV generation. These IDs -are embedded in test labels and flow into the dashboard CSV. The export step -later discovers the marker via :func:`_discover_precreated_mlflow_run_id` and -resumes the same run (``mlflow.start_run(run_id=...)``) to upload artifacts, -avoiding a duplicate run. - -The marker is a YAML file with keys ``run_id`` and ``experiment_id``, written -to ``ARTIFACT_DIR`` by :func:`precreate_mlflow_run`. -""" +TEST_LABELS_FILENAME = "__test_labels__.yaml" def precreate_mlflow_run( @@ -288,13 +286,13 @@ def precreate_mlflow_run( experiment: str | None = None, workspace: str | None = None, ) -> dict[str, str]: - """Pre-create an MLflow run and write the marker file to ``ARTIFACT_DIR``. + """Pre-create an MLflow run so the export step can resume it. The run is created and immediately ended (status FINISHED). The export step will resume it via ``mlflow.start_run(run_id=...)`` to upload artifacts. - The caller (test harness) is responsible for reading config and vault paths; - this function does not access the project config directly. + The caller is responsible for persisting the returned IDs (e.g. via the + ``mlflow_destination`` section of ``__test_labels__.yaml``). Returns a dict with ``run_id`` and ``experiment_id``. """ @@ -333,35 +331,30 @@ def precreate_mlflow_run( meta = {"run_id": run_id, "experiment_id": experiment_id} - marker_path = env.ARTIFACT_DIR / MLFLOW_PRECREATED_RUN_MARKER - marker_path.parent.mkdir(parents=True, exist_ok=True) - marker_path.write_text(yaml.safe_dump(meta, sort_keys=False), encoding="utf-8") - logger.info( - "Pre-created MLflow run %s (experiment=%s), marker: %s", - run_id, - experiment_id, - marker_path, - ) + logger.info("Pre-created MLflow run %s (experiment=%s)", run_id, experiment_id) return meta -def _read_mlflow_ids_from_marker() -> tuple[str, str]: - """Read run_id and experiment_id from the pre-created marker file on disk.""" +def _read_mlflow_ids_from_test_labels() -> tuple[str, str]: + """Read run_id and experiment_id from ``mlflow_destination`` in test labels.""" artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None if not artifact_dir: - logger.warning("ARTIFACT_DIR not set, cannot read MLflow marker") + logger.warning("ARTIFACT_DIR not set, cannot read MLflow destination from test labels") return "", "" - for marker in sorted(artifact_dir.rglob(MLFLOW_PRECREATED_RUN_MARKER)): - data = yaml.safe_load(marker.read_text(encoding="utf-8")) - if not isinstance(data, dict): - logger.warning( - "MLflow marker %s has unexpected type %s, skipping", marker, type(data).__name__ - ) - continue - if data.get("run_id"): - return data["run_id"], data.get("experiment_id", "") - logger.warning("No valid MLflow marker found under %s", artifact_dir) + for labels_file in sorted(artifact_dir.rglob(TEST_LABELS_FILENAME)): + try: + data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) + if not isinstance(data, dict): + continue + dest = data.get("mlflow_destination") + if not isinstance(dest, dict): + continue + run_id = dest.get("run_id", "") + if run_id: + return run_id, dest.get("experiment_id", "") + except (OSError, yaml.YAMLError) as e: + logger.warning("Failed to read test labels %s: %s", labels_file, e) return "", "" @@ -382,9 +375,9 @@ def build_mlflow_run_url( load_mlflow_secrets_yaml, ) - run_id, experiment_id = _read_mlflow_ids_from_marker() + run_id, experiment_id = _read_mlflow_ids_from_test_labels() if not run_id or not experiment_id: - logger.warning("Cannot build MLflow URL: run_id or experiment_id missing from marker") + logger.warning("Cannot build MLflow URL: run_id or experiment_id missing from test labels") return "" if not secrets_path.exists(): @@ -403,22 +396,20 @@ def build_mlflow_run_url( def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: - """Find a pre-created MLflow run_id from the marker file.""" - for marker in sorted(from_path.rglob(MLFLOW_PRECREATED_RUN_MARKER)): + """Find a pre-created MLflow run_id from ``mlflow_destination`` in test labels.""" + for labels_file in sorted(from_path.rglob(TEST_LABELS_FILENAME)): try: - data = yaml.safe_load(marker.read_text(encoding="utf-8")) + data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) if not isinstance(data, dict): - logger.warning( - "MLflow marker %s has unexpected type %s, skipping", - marker, - type(data).__name__, - ) continue - run_id = data.get("run_id") + dest = data.get("mlflow_destination") + if not isinstance(dest, dict): + continue + run_id = dest.get("run_id") if run_id: - logger.info("Found pre-created MLflow run_id: %s (from marker %s)", run_id, marker) + logger.info("Found pre-created MLflow run_id: %s (from %s)", run_id, labels_file) return run_id except (OSError, yaml.YAMLError) as e: - logger.warning("Failed to read MLflow pre-created run marker %s: %s", marker, e) + logger.warning("Failed to read test labels %s: %s", labels_file, e) return None diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 8eccb5af4..9a2f9201d 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -39,6 +39,7 @@ def write_test_labels( version: str = "1", dump_config: bool = True, kpi_labels: dict[str, str] | None = None, + mlflow_destination: dict[str, str] | None = None, ) -> Path: """Write a __test_labels__.yaml file to mark a directory as a Caliper test base. @@ -48,6 +49,7 @@ def write_test_labels( version: Version string for the test labels format (default: "1") dump_config: Whether to save project configuration to config.yaml (default: True) kpi_labels: Optional dictionary of KPI labels for system context + mlflow_destination: Optional MLflow run destination (run_id, experiment_id, workspace) Returns: Path to the created __test_labels__.yaml file @@ -75,6 +77,9 @@ def write_test_labels( if kpi_labels: payload["kpi_labels"] = kpi_labels + if mlflow_destination: + payload["mlflow_destination"] = mlflow_destination + # Create directory and write YAML test_labels_path.parent.mkdir(parents=True, exist_ok=True) with test_labels_path.open("w", encoding="utf-8") as handle: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 232938c68..cf9b641b1 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -9,7 +9,6 @@ import yaml from projects.core.library import env -from projects.core.library import vault as vault_lib from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.rhaiis.orchestration import runtime_config @@ -299,44 +298,7 @@ def _run_test( logger.exception("Profiler trace upload failed") _warnings.append("Profiler trace upload failed") - # Pre-create MLflow run so its URL can be embedded in test labels / CSV - mlflow_run_meta: dict[str, str] = {} - try: - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", - None, - print=False, - warn=False, - ) - if vault_name and vault_key: - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if secrets_path and secrets_path.exists(): - from projects.caliper.orchestration.export import precreate_mlflow_run - - mlflow_run_meta = precreate_mlflow_run( - secrets_path=secrets_path, - experiment=config.project.get_config( - "caliper.export.backend.mlflow.config.experiment", - None, - print=False, - warn=False, - ), - workspace=config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", - None, - print=False, - warn=False, - ), - ) - else: - logger.info("MLflow secrets file not found, skipping run pre-creation") - else: - logger.info("MLflow vault not configured, skipping run pre-creation") - except Exception: - logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + mlflow_run_meta = _precreate_mlflow_run_if_configured() # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None @@ -359,8 +321,7 @@ def _run_test( version=version, cluster_tag=cluster_tag, trtllm_config=trtllm_cfg, - mlflow_run_id=mlflow_run_meta.get("run_id", ""), - mlflow_experiment_id=mlflow_run_meta.get("experiment_id", ""), + mlflow_run_meta=mlflow_run_meta or None, ) try: @@ -427,8 +388,7 @@ def _run_workload_benchmark( version: str, cluster_tag: str, trtllm_config: dict | None = None, - mlflow_run_id: str = "", - mlflow_experiment_id: str = "", + mlflow_run_meta: dict[str, str] | None = None, ) -> None: """Run benchmark and post-processing for a single workload. @@ -462,8 +422,7 @@ def _run_workload_benchmark( accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, trtllm_config=trtllm_config, - mlflow_run_id=mlflow_run_id, - mlflow_experiment_id=mlflow_experiment_id, + mlflow_run_meta=mlflow_run_meta, ) if not run_benchmark: @@ -507,6 +466,48 @@ def _run_workload_benchmark( ) +def _precreate_mlflow_run_if_configured() -> dict[str, str]: + """Pre-create an MLflow run so its IDs can be embedded in test labels. + + Uses the guard pattern with early returns to avoid deep nesting. + Returns a dict with run_id and experiment_id, or empty dict on failure. + """ + from projects.core.library import config + from projects.core.library import vault as vault_lib + + vault_name = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False + ) + vault_key = config.project.get_config( + "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False + ) + if not vault_name or not vault_key: + logger.info("MLflow vault not configured, skipping run pre-creation") + return {} + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.info("MLflow secrets file not found, skipping run pre-creation") + return {} + + experiment = config.project.get_config( + "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False + ) + workspace = config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False + ) + + try: + from projects.caliper.orchestration.export import precreate_mlflow_run + + return precreate_mlflow_run( + secrets_path=secrets_path, experiment=experiment, workspace=workspace + ) + except Exception: + logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + return {} + + def _create_test_labels( model_key: str, workload_key: str, @@ -520,8 +521,7 @@ def _create_test_labels( accelerator_chip: str = "", run_uuid: str = "", trtllm_config: dict | None = None, - mlflow_run_id: str = "", - mlflow_experiment_id: str = "", + mlflow_run_meta: dict[str, str] | None = None, ) -> None: _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") parts = [f"{k}: {v}" for k, v in engine_args.items()] @@ -548,10 +548,21 @@ def _create_test_labels( "cluster_tag": cluster_tag, "runtime_args": runtime_args, "run_uuid": run_uuid, - "mlflow_run_id": mlflow_run_id, - "mlflow_experiment_id": mlflow_experiment_id, } - write_test_labels(env.ARTIFACT_DIR, labels) + + mlflow_destination = None + if mlflow_run_meta and mlflow_run_meta.get("run_id"): + from projects.core.library import config + + mlflow_destination = { + "run_id": mlflow_run_meta["run_id"], + "experiment_id": mlflow_run_meta.get("experiment_id", ""), + "workspace": config.project.get_config( + "caliper.export.backend.mlflow.config.workspace", "", print=False, warn=False + ), + } + + write_test_labels(env.ARTIFACT_DIR, labels, mlflow_destination=mlflow_destination) logger.info("Created test labels: %s", labels) diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 803cbf68a..9071b946d 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -18,6 +18,30 @@ logger = logging.getLogger(__name__) + +def _read_mlflow_destination_from_test_labels(kpi_records: list[dict]) -> dict[str, str]: + """Extract mlflow_destination from __test_labels__.yaml near the first KPI run_path.""" + import yaml + + for kpi in kpi_records: + run_path = kpi.get("run_path", "") + if not run_path: + continue + labels_file = Path(run_path) / "__test_labels__.yaml" + if not labels_file.exists(): + labels_file = Path(run_path).parent / "__test_labels__.yaml" + if not labels_file.exists(): + continue + try: + data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) + dest = data.get("mlflow_destination") if isinstance(data, dict) else None + if isinstance(dest, dict) and dest.get("run_id"): + return dest + except (OSError, yaml.YAMLError): + pass + return {} + + # CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. # GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. _SECONDS_TO_MS_COLUMNS = frozenset( @@ -128,6 +152,8 @@ def export_kpis_to_csv( """ from projects.rhaiis.postprocess.csv_export import FIELDNAMES + mlflow_dest = _read_mlflow_destination_from_test_labels(kpi_records) + # Group KPIs by (run_path, rate_index) groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(dict) group_labels: dict[tuple[str, str], dict[str, Any]] = {} @@ -191,8 +217,8 @@ def export_kpis_to_csv( row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") row["guidellm_version"] = labels.get("guidellm_version", "") - row["mlflow_run_id"] = labels.get("mlflow_run_id", "") - row["mlflow_experiment_id"] = labels.get("mlflow_experiment_id", "") + row["mlflow_run_id"] = mlflow_dest.get("run_id", "") + row["mlflow_experiment_id"] = mlflow_dest.get("experiment_id", "") rows.append(row) output_path.parent.mkdir(parents=True, exist_ok=True) From bcb70f078906fd8801d1d9022abba43cebe6d466 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Wed, 5 Aug 2026 18:00:13 +0300 Subject: [PATCH 0436/1072] Rename metrics_from_kpis.py to kpis_to_mlflow.py Consistent with the source_to_destination naming convention used elsewhere: artifacts_to_kpis, kpis_to_csv, artifacts_to_ai_data. Co-authored-by: Cursor --- .../engine/kpi/{metrics_from_kpis.py => kpis_to_mlflow.py} | 0 projects/caliper/orchestration/postprocess.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename projects/caliper/engine/kpi/{metrics_from_kpis.py => kpis_to_mlflow.py} (100%) diff --git a/projects/caliper/engine/kpi/metrics_from_kpis.py b/projects/caliper/engine/kpi/kpis_to_mlflow.py similarity index 100% rename from projects/caliper/engine/kpi/metrics_from_kpis.py rename to projects/caliper/engine/kpi/kpis_to_mlflow.py diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index e782ef5e7..232e2df7d 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1075,7 +1075,7 @@ def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: return try: - from projects.caliper.engine.kpi.metrics_from_kpis import generate_metrics_from_kpis + from projects.caliper.engine.kpi.kpis_to_mlflow import generate_metrics_from_kpis result = generate_metrics_from_kpis(kpis_json_path, self.tree_root) status = result.get("status", "unknown") From 22d9bfbc82243f510373cabaeb80fb33e54f00bc Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 5 Aug 2026 15:11:53 -0400 Subject: [PATCH 0437/1072] [rhaiis] Add presets for benchmark, cluster, and model aliases Add new preset files to reduce boilerplate in FournosJob configs: - benchmarks.yaml: common benchmark settings (run_benchmark, profiler, dashboard, agent_analysis, slack_notify_always) - clusters.yaml: hera/zeus cluster configs (cluster_tag, image_pull_secrets, fs_group) - presets.yaml: expand model aliases to cover all models in models.yaml with short names defaulting to FP8 variants, add ci-quick preset with qwen3-0.6b/TP1 for fast validation runs Also remove unused top-level images fallback from rhaiis.yaml since all engines define their own images, and delete stale mehulvalidation.yaml preset. Co-authored-by: Cursor --- .../rhaiis/orchestration/config.d/rhaiis.yaml | 6 +- .../orchestration/config.d/workloads.yaml | 2 +- projects/rhaiis/orchestration/config.yaml | 2 + .../orchestration/presets.d/benchmarks.yaml | 8 + .../orchestration/presets.d/clusters.yaml | 11 + .../presets.d/mehulvalidation.yaml | 3 - .../orchestration/presets.d/presets.yaml | 222 +++++++++++++++--- 7 files changed, 208 insertions(+), 46 deletions(-) create mode 100644 projects/rhaiis/orchestration/presets.d/benchmarks.yaml create mode 100644 projects/rhaiis/orchestration/presets.d/clusters.yaml delete mode 100644 projects/rhaiis/orchestration/presets.d/mehulvalidation.yaml diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 89b904e48..358f710b1 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -7,10 +7,6 @@ gpu_types: amd: mi300x l40s: l40s -images: - nvidia: quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2-1773886296 - amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5-1766067105 - engine: vllm engines: @@ -94,4 +90,4 @@ profiler: agent_analysis: enabled: false severity_threshold: 10 - url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" + url: "" diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 833cbb5e0..4594fe9d6 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -1,5 +1,5 @@ ci-quick: - data: "prompt_tokens=256,output_tokens=128" + data: "prompt_tokens=1000,output_tokens=1000" rates: [1, 5] max_seconds: 60 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index a5dffe14f..dd7206936 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -12,6 +12,7 @@ benchmarks: timeout: 14400 pvc_size: 10Gi hf_token_secret: "storage-config" + fs_group: null args: backend_type: openai_http rate_type: concurrent @@ -28,6 +29,7 @@ tests: version: "" compare_version: "" slack_user: "" + slack_notify_always: false caliper: postprocess: diff --git a/projects/rhaiis/orchestration/presets.d/benchmarks.yaml b/projects/rhaiis/orchestration/presets.d/benchmarks.yaml new file mode 100644 index 000000000..fdb3e2070 --- /dev/null +++ b/projects/rhaiis/orchestration/presets.d/benchmarks.yaml @@ -0,0 +1,8 @@ +__multiple: true + +benchmark: + tests.rhaiis.run_benchmark: true + tests.rhaiis.slack_notify_always: true + rhaiis.profiler.enabled: true + caliper.postprocess.csv_dashboard.enabled: true + rhaiis.agent_analysis.enabled: false diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml new file mode 100644 index 000000000..acf09c811 --- /dev/null +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -0,0 +1,11 @@ +__multiple: true + +hera: + rhaiis.cluster_tag: "hera2" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 + +zeus: + rhaiis.cluster_tag: "zeus2" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 diff --git a/projects/rhaiis/orchestration/presets.d/mehulvalidation.yaml b/projects/rhaiis/orchestration/presets.d/mehulvalidation.yaml deleted file mode 100644 index bee5b6432..000000000 --- a/projects/rhaiis/orchestration/presets.d/mehulvalidation.yaml +++ /dev/null @@ -1,3 +0,0 @@ -rhaiis.namespace: kserve-e2e-perf -rhaiis.deploy.image_pull_secret: npalaska-image-pull -rhaiis.deploy.service_account_name: sa diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index a2ed40b80..bb110295b 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -1,33 +1,137 @@ __multiple: true +# ── Accelerator ────────────────────────────────────────────── + +nvidia: + rhaiis.accelerator: nvidia + +amd: + rhaiis.accelerator: amd + +# ── Engine ─────────────────────────────────────────────────── + +vllm: + rhaiis.engine: vllm + +sglang: + rhaiis.engine: sglang + +trtllm: + rhaiis.engine: trtllm + +# ── Workload profiles ─────────────────────────────────────── + +ci-quick: + tests.rhaiis.workload_key: ci-quick + tests.rhaiis.model_key: qwen3-0_6b + tests.rhaiis.warmup: false + tests.rhaiis.version: "vLLM-0.24.0-ci-test" + rhaiis.profiler.enabled: false + caliper.postprocess.csv_dashboard.enabled: false + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +profile1: + tests.rhaiis.workload_key: profile1 + +profile2: + tests.rhaiis.workload_key: profile2 + +profile3: + tests.rhaiis.workload_key: profile3 + +profile4: + tests.rhaiis.workload_key: profile4 + +# ── Models ────────────────────────────────────────────────── +# Short aliases default to the FP8 variant where available. +# Use the full name (e.g. llama-70b-w4a16) for other quants. + +# Validation model +qwen3-0.6b: + tests.rhaiis.model_key: qwen3-0_6b + # Llama 3.1 8B llama-8b: - tests.rhaiis.model_key: llama-3-1-8b-fp8 + tests.rhaiis.model_key: llama-3-1-8b # Llama 3.3 70B llama-70b: tests.rhaiis.model_key: llama-3-3-70b-fp8 +llama-70b-bf16: + tests.rhaiis.model_key: llama-3-3-70b + +llama-70b-w8a8: + tests.rhaiis.model_key: llama-3-3-70b-w8a8 + +llama-70b-w4a16: + tests.rhaiis.model_key: llama-3-3-70b-w4a16 + # Llama 3.1 405B llama-405b: tests.rhaiis.model_key: llama-3-1-405b-fp8 -# Llama 4 Scout +# Llama 4 Scout (MoE) llama-4-scout: tests.rhaiis.model_key: llama-4-scout-fp8 -# Llama 4 Maverick +llama-4-scout-bf16: + tests.rhaiis.model_key: llama-4-scout + +llama-4-scout-int4: + tests.rhaiis.model_key: llama-4-scout-int4 + +# Llama 4 Maverick (MoE) llama-4-maverick: tests.rhaiis.model_key: llama-4-maverick-fp8 -# Granite 8B +llama-4-maverick-bf16: + tests.rhaiis.model_key: llama-4-maverick + +llama-4-maverick-w4a16: + tests.rhaiis.model_key: llama-4-maverick-w4a16 + +# Granite 3.1 8B granite-8b: tests.rhaiis.model_key: granite-3-1-8b-fp8 -# Mistral Small 3.1 +granite-8b-bf16: + tests.rhaiis.model_key: granite-3-1-8b-instruct + +granite-8b-w4a16: + tests.rhaiis.model_key: granite-3-1-8b-w4a16 + +granite-8b-w8a8: + tests.rhaiis.model_key: granite-3-1-8b-w8a8 + +# Mistral Small 3.1 24B mistral-24b: tests.rhaiis.model_key: mistral-2503-fp8 +mistral-24b-bf16: + tests.rhaiis.model_key: mistral-2503 + +mistral-24b-w4a16: + tests.rhaiis.model_key: mistral-2503-w4a16 + +mistral-24b-w8a8: + tests.rhaiis.model_key: mistral-2503-w8a8 + +# Mistral 7B +mistral-7b: + tests.rhaiis.model_key: mistral-7b + +# Ministral 3 14B +ministral-14b: + tests.rhaiis.model_key: ministral-3-14b-fp8 + +ministral-14b-bf16: + tests.rhaiis.model_key: ministral-3-14b-fp16 + +ministral-14b-reasoning: + tests.rhaiis.model_key: ministral-3-14b-reasoning + # Phi 4 phi-4: tests.rhaiis.model_key: phi-4-fp8 @@ -36,52 +140,96 @@ phi-4: qwen25-7b: tests.rhaiis.model_key: qwen25-7b-fp8 -# Qwen3 235B +qwen25-7b-bf16: + tests.rhaiis.model_key: qwen25-7b-instruct + +qwen25-7b-w8a8: + tests.rhaiis.model_key: qwen25-7b-w8a8 + +qwen25-7b-w4a16: + tests.rhaiis.model_key: qwen25-7b-w4a16 + +# Qwen3 235B (MoE) qwen3-235b: tests.rhaiis.model_key: qwen3-235b-instruct-fp8 -# DeepSeek R1 -deepseek-r1: - tests.rhaiis.model_key: deepseek-r1-0528 +qwen3-235b-bf16: + tests.rhaiis.model_key: qwen3-235b-instruct -# DeepSeek V3.2 -deepseek-v3: - tests.rhaiis.model_key: deepseek-v3-2 +qwen3-235b-fp8-dynamic: + tests.rhaiis.model_key: qwen3-235b-fp8-dynamic -# GPT-OSS 120B +# Qwen3 30B (MoE) +qwen3-30b: + tests.rhaiis.model_key: qwen3-30b-a3b-instruct + +qwen3-30b-a3b: + tests.rhaiis.model_key: qwen3-30b-a3b + +# Qwen3 80B (MoE) +qwen3-80b: + tests.rhaiis.model_key: qwen3-n-80b-a3b-instruct + +# Qwen3 VL +qwen3-vl-235b: + tests.rhaiis.model_key: qwen3-vl-235b-fp8 + +qwen3-vl-30b: + tests.rhaiis.model_key: qwen3-vl-30b-a3b + +# GPT-OSS gpt-oss: tests.rhaiis.model_key: gpt-oss-120b-fp8 -# CI quick validation -ci-quick: - tests.rhaiis.workload_key: ci-quick +gpt-oss-bf16: + tests.rhaiis.model_key: gpt-oss-120b -# Workload profiles -profile1: - tests.rhaiis.workload_key: profile1 +gpt-oss-20b: + tests.rhaiis.model_key: gpt-oss-20b -profile2: - tests.rhaiis.workload_key: profile2 +# Nemotron 3.1 70B +nemotron-70b: + tests.rhaiis.model_key: nemotron-3-1-70b -profile3: - tests.rhaiis.workload_key: profile3 +nemotron-70b-fp8: + tests.rhaiis.model_key: nemotron-3-1-70b-fp8 -profile4: - tests.rhaiis.workload_key: profile4 +# Nemotron 3 Nano 30B (MoE) +nemotron3nano-30b: + tests.rhaiis.model_key: nemotron3nano-30b -# Accelerator -nvidia: - rhaiis.accelerator: nvidia +nemotron3nano-30b-fp8: + tests.rhaiis.model_key: nemotron3nano-30b-fp8 -amd: - rhaiis.accelerator: amd +# Nemotron 3 Super 120B (MoE) +nemotron3super-120b: + tests.rhaiis.model_key: nemotron3super-120b-fp8 -# Engine -vllm: - rhaiis.engine: vllm +# Mixtral 8x7B +mixtral: + tests.rhaiis.model_key: mixtral-8x7b -sglang: - rhaiis.engine: sglang +# Gemma 2 9B +gemma-9b: + tests.rhaiis.model_key: gemma-9b-fp8 -trtllm: - rhaiis.engine: trtllm +gemma-9b-bf16: + tests.rhaiis.model_key: gemma-9b + +gemma-9b-w8a16: + tests.rhaiis.model_key: gemma-9b-w8a16 + +# DeepSeek R1 +deepseek-r1: + tests.rhaiis.model_key: deepseek-r1-0528 + +deepseek-r1-w4a16: + tests.rhaiis.model_key: deepseek-r1-0528-w4a16 + +# DeepSeek V3.2 +deepseek-v3: + tests.rhaiis.model_key: deepseek-v3-2 + +# DeepSeek V4 Pro +deepseek-v4-pro: + tests.rhaiis.model_key: deepseek-v4-pro From e32e1d4e743dd1b96143ef0ea8a1cb5a015bf6da Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 6 Aug 2026 11:01:05 +0300 Subject: [PATCH 0438/1072] Add caliper kpi kpis-to-mlflow CLI command and fix encapsulation Replace direct engine import in postprocess.py with fork/exec subprocess call via the new `caliper kpi kpis-to-mlflow` CLI command, matching the pattern used by all other Caliper orchestration steps. - Add kpis-to-mlflow click command in cli/commands.py - Add build_kpis_to_mlflow_command in cli_builder.py - Register command under kpi_group in cli/main.py - Orchestrator catches step failure and records status, continuing remaining steps - kpis_to_mlflow.py raises FileNotFoundError instead of swallowing Co-authored-by: Cursor --- projects/caliper/cli/commands.py | 70 +++++++++++++++++++ projects/caliper/cli/main.py | 2 + projects/caliper/engine/kpi/kpis_to_mlflow.py | 12 ++-- projects/caliper/orchestration/cli_builder.py | 24 +++++++ projects/caliper/orchestration/postprocess.py | 62 ++++++++++------ 5 files changed, 141 insertions(+), 29 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 939c927fb..e2135b536 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -695,6 +695,76 @@ def kpi_csv_export( sys.exit(3) +@click.command("kpis-to-mlflow") +@click.option( + "--input", + "input_file", + type=click.Path(path_type=Path), + required=True, + help="Input KPI JSON file (schema v2 hierarchical format)", +) +@click.option( + "--artifacts-dir", + "artifacts_dir", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Root of the artifact tree containing __test_labels__.yaml markers", +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +def kpis_to_mlflow_cmd( + input_file: Path, + artifacts_dir: Path, + status_file: Path | None, +) -> None: + """Convert kpis.json into per-run metrics.json + parameters.json for MLflow.""" + from projects.caliper.engine.kpi.kpis_to_mlflow import generate_metrics_from_kpis + + status_data: dict = {"success": False} + + try: + result = generate_metrics_from_kpis(input_file, artifacts_dir) + status = result.get("status", "unknown") + if status == "success": + status_data = { + "success": True, + "tests_processed": result.get("tests_processed", 0), + "total_tests": result.get("total_tests", 0), + } + click.echo( + f"Generated metrics.json for {result.get('tests_processed', 0)}/" + f"{result.get('total_tests', 0)} test(s)" + ) + elif status == "skipped": + status_data = {"success": True, "skipped": True, "reason": result.get("reason", "")} + click.echo(f"Skipped: {result.get('reason', '')}") + else: + status_data = {"success": False, "error": result.get("error", "unknown error")} + click.echo(f"kpis-to-mlflow failed: {result.get('error', 'unknown')}", err=True) + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"kpis-to-mlflow failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + if not status_data.get("success", False): + sys.exit(3) + + @click.command("import") @click.option("--snapshot", type=click.Path(path_type=Path), required=True) @click.pass_context diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index f3e16902d..0bfcd2c13 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -18,6 +18,7 @@ kpi_generate, kpi_import, kpi_s3_import, + kpis_to_mlflow_cmd, list_reports_cmd, parse_cmd, visualize_cmd, @@ -212,6 +213,7 @@ def run_cli() -> None: kpi_group.add_command(kpi_import) kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) +kpi_group.add_command(kpis_to_mlflow_cmd) # Register s3-export command under kpi group kpi_group.add_command(s3_export_cmd) diff --git a/projects/caliper/engine/kpi/kpis_to_mlflow.py b/projects/caliper/engine/kpi/kpis_to_mlflow.py index 77848bff0..eb358fae3 100644 --- a/projects/caliper/engine/kpi/kpis_to_mlflow.py +++ b/projects/caliper/engine/kpi/kpis_to_mlflow.py @@ -93,14 +93,10 @@ def generate_metrics_from_kpis( Status dict with counts and any warnings. """ if not kpis_json_path.is_file(): - return {"status": "skipped", "reason": f"kpis.json not found: {kpis_json_path}"} - - try: - with kpis_json_path.open(encoding="utf-8") as f: - data = json.load(f) - except (json.JSONDecodeError, OSError) as e: - logger.warning("Failed to read kpis.json at %s: %s", kpis_json_path, e) - return {"status": "failed", "error": str(e)} + raise FileNotFoundError(f"kpis.json not found: {kpis_json_path}") + + with kpis_json_path.open(encoding="utf-8") as f: + data = json.load(f) if not isinstance(data, dict) or data.get("schema_version") != "2": return {"status": "skipped", "reason": "Not a schema v2 kpis.json"} diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py index 58daa50cd..ee7d0294d 100644 --- a/projects/caliper/orchestration/cli_builder.py +++ b/projects/caliper/orchestration/cli_builder.py @@ -299,6 +299,30 @@ def build_ai_eval_export_command( return cmd +def build_kpis_to_mlflow_command( + tree_root: Path, + status_file: Path, + input_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi kpis-to-mlflow. + + Args: + tree_root: Root of the artifact tree with __test_labels__.yaml markers + status_file: Where to write status YAML + input_file: Input kpis.json file (schema v2) + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["kpi", "kpis-to-mlflow"] + + cmd.extend(["--input", str(input_file)]) + cmd.extend(["--artifacts-dir", str(tree_root)]) + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + def build_s3_import_command( config: CaliperOrchestrationPostprocessConfig, status_file: Path, diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 232e2df7d..48d5f53c9 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -27,6 +27,7 @@ build_analyse_kpis_command, build_kpi_csv_export_command, build_kpi_generate_command, + build_kpis_to_mlflow_command, build_parse_command, build_s3_export_command, build_s3_import_command, @@ -1017,7 +1018,14 @@ def _run_kpi_and_ai_data_steps(self) -> None: self._run_artifacts_to_kpis_step(output_dir, mod_str) # Generate per-run metrics.json + parameters.json from kpis.json - self._run_kpis_to_metrics_step(output_dir) + try: + self._run_kpis_to_metrics_step(output_dir) + except Exception as e: + logger.error("kpis-to-mlflow step failed: %s", e) + self._add_step( + "kpis_to_mlflow", + {"status": "failed", "error": str(e), "completed_at": time.time()}, + ) # KPI CSV export self._run_kpis_to_csv_step(output_dir) @@ -1061,35 +1069,47 @@ def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: """Generate per-run metrics.json + parameters.json from kpis.json. - Runs automatically after kpis.json generation succeeds. Writes - files into each test run directory so the MLflow export backend - picks them up via ``_log_metrics_and_params_from_tree``. + Runs automatically after kpis.json generation succeeds. Uses + ``caliper kpi kpis-to-mlflow`` via fork/exec like all other steps. """ kpi_step = self._get_step("artifacts_to_kpis") if not kpi_step or kpi_step.get("status") != "success": return kpis_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - if not kpis_json_path.is_file(): - logger.warning("kpis.json not found at %s, skipping metrics generation", kpis_json_path) - return + + status_file = output_dir / "kpis_to_mlflow_status.yaml" + + command = build_kpis_to_mlflow_command( + tree_root=self.tree_root, + status_file=status_file, + input_file=kpis_json_path, + ) + + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi kpis-to-mlflow", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) try: - from projects.caliper.engine.kpi.kpis_to_mlflow import generate_metrics_from_kpis + status_file.unlink() + except FileNotFoundError: + pass - result = generate_metrics_from_kpis(kpis_json_path, self.tree_root) - status = result.get("status", "unknown") - if status == "success": - logger.info( - "kpis-to-metrics: wrote metrics.json for %d test(s)", - result.get("tests_processed", 0), - ) - elif status == "skipped": - logger.info("kpis-to-metrics: skipped — %s", result.get("reason", "")) - else: - logger.warning("kpis-to-metrics: %s", result.get("error", "unknown error")) - except Exception as e: - logger.warning("kpis-to-metrics conversion failed (non-fatal): %s", e) + step_result = { + "status": "success" if status_data.get("success") else "failed", + "completed_at": time.time(), + } + if status_data.get("success"): + step_result["tests_processed"] = status_data.get("tests_processed", 0) + step_result["total_tests"] = status_data.get("total_tests", 0) + self._add_step("kpis_to_mlflow", step_result, log_file) + + if result.returncode != 0 or not status_data.get("success"): + error = status_data.get("error", f"exit code {result.returncode}") + raise RuntimeError(f"kpis-to-mlflow failed: {error}") def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" From 97d86812d9ca33341d93f5a2bfba6140b07089ab Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 6 Aug 2026 11:31:08 +0300 Subject: [PATCH 0439/1072] address reviews --- module/skills/forge-review/SKILL.md | 8 ++++++++ .../skills/forge-review/scripts/gather-diff.sh | 16 ++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/module/skills/forge-review/SKILL.md b/module/skills/forge-review/SKILL.md index e09153cfa..1bdb40096 100644 --- a/module/skills/forge-review/SKILL.md +++ b/module/skills/forge-review/SKILL.md @@ -62,6 +62,14 @@ For every added or modified line in the diff (`+` lines), check against each rul - Any file under `projects/*/toolbox/` importing from `projects/*/orchestration/` - Toolbox code reading config via `config.project.get_config()` directly instead of receiving values through `run()` parameters - Manifest/template files placed outside their toolbox command directory +- **Caliper orchestration vs engine**: The `projects/caliper/engine/` layer contains pure + data-processing logic (parsing artifacts, building the unified model, computing KPIs, + rendering visualizations). The `projects/caliper/orchestration/` layer drives the engine + via CLI subprocess calls and handles CI concerns (step sequencing, status files, logging, + notifications, final status computation). Orchestration code must not duplicate or inline + engine logic — e.g. directly traversing artifact trees, computing KPI values, rendering + plots, or manipulating the unified model. If new data-processing capability is needed, + add it to the engine and expose it through the CLI; orchestration only invokes it. #### No time.sleep() - `time.sleep(N)` used to wait for cluster state (pod ready, operator deployed, resource status) diff --git a/module/skills/forge-review/scripts/gather-diff.sh b/module/skills/forge-review/scripts/gather-diff.sh index c93cf6488..ce84ef61e 100755 --- a/module/skills/forge-review/scripts/gather-diff.sh +++ b/module/skills/forge-review/scripts/gather-diff.sh @@ -2,15 +2,15 @@ set -euo pipefail # Gather the diff that represents what the user intends to commit. -# Priority: staged > unstaged > branch diff against remote. +# Priority: staged > unstaged > branch diff against main (what the PR proposes). # Outputs structured sections so the reviewing agent can parse reliably. BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "HEAD") -REMOTE_BRANCH="origin/${BRANCH}" +BASE_BRANCH="main" echo "=== DIFF_META ===" echo "branch: ${BRANCH}" -echo "remote: ${REMOTE_BRANCH}" +echo "base: ${BASE_BRANCH}" echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "" @@ -36,9 +36,9 @@ elif [[ -n "${UNSTAGED}" ]]; then echo "=== DIFF_TYPE: unstaged ===" echo "${UNSTAGED}" else - HAS_REMOTE=$(git ls-remote --heads origin "${BRANCH}" 2>/dev/null | head -1) - if [[ -n "${HAS_REMOTE}" ]]; then - BRANCH_DIFF=$(git diff "${REMOTE_BRANCH}...HEAD" 2>/dev/null || true) + MERGE_BASE=$(git merge-base "${BASE_BRANCH}" HEAD 2>/dev/null || true) + if [[ -n "${MERGE_BASE}" ]]; then + BRANCH_DIFF=$(git diff "${MERGE_BASE}...HEAD" 2>/dev/null || true) if [[ -n "${BRANCH_DIFF}" ]]; then echo "" echo "=== DIFF_TYPE: branch ===" @@ -46,12 +46,12 @@ else else echo "" echo "=== DIFF_TYPE: none ===" - echo "No changes found between local and remote." + echo "No changes found between ${BASE_BRANCH} and HEAD." fi else echo "" echo "=== DIFF_TYPE: none ===" - echo "No remote branch and no local changes." + echo "Could not determine merge-base with ${BASE_BRANCH}." fi fi From 8543da1296dd65721176fb4f991055722e4c30fe Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 6 Aug 2026 11:43:21 +0300 Subject: [PATCH 0440/1072] Move kpis-to-mlflow error handling inside the step method Self-contained failure handling in _run_kpis_to_metrics_step, consistent with all other _run_*_step methods. Co-authored-by: Cursor --- projects/caliper/orchestration/postprocess.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 48d5f53c9..cc2ff1bed 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1018,14 +1018,7 @@ def _run_kpi_and_ai_data_steps(self) -> None: self._run_artifacts_to_kpis_step(output_dir, mod_str) # Generate per-run metrics.json + parameters.json from kpis.json - try: - self._run_kpis_to_metrics_step(output_dir) - except Exception as e: - logger.error("kpis-to-mlflow step failed: %s", e) - self._add_step( - "kpis_to_mlflow", - {"status": "failed", "error": str(e), "completed_at": time.time()}, - ) + self._run_kpis_to_metrics_step(output_dir) # KPI CSV export self._run_kpis_to_csv_step(output_dir) @@ -1109,7 +1102,7 @@ def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: if result.returncode != 0 or not status_data.get("success"): error = status_data.get("error", f"exit code {result.returncode}") - raise RuntimeError(f"kpis-to-mlflow failed: {error}") + logger.error("kpis-to-mlflow step failed: %s", error) def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" From 78a8b7444a8d394eced0641268ca4cd310a756c8 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Thu, 6 Aug 2026 11:49:48 +0300 Subject: [PATCH 0441/1072] address coderabit --- module/README.md | 4 ++-- module/skills/forge-review/SKILL.md | 5 +++-- module/skills/forge-review/scripts/gather-diff.sh | 11 +++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/module/README.md b/module/README.md index d371b8467..5341761ad 100644 --- a/module/README.md +++ b/module/README.md @@ -108,14 +108,14 @@ This installs the module into whichever agents they have configured. If you prefer not to use Lola, copy the skill directory manually. -#### Cursor +### Cursor ```bash mkdir -p .cursor/skills/forge-review cp -r module/skills/forge-review/* .cursor/skills/forge-review/ ``` -#### Claude Code +### Claude Code ```bash mkdir -p .claude/skills/forge-review diff --git a/module/skills/forge-review/SKILL.md b/module/skills/forge-review/SKILL.md index 1bdb40096..cf7fdd26f 100644 --- a/module/skills/forge-review/SKILL.md +++ b/module/skills/forge-review/SKILL.md @@ -18,10 +18,11 @@ Run this review **before every commit**. If the user asks to commit, run this re ### Step 1: Gather the diff -Run the diff gathering script: +Run the diff gathering script that lives alongside this SKILL.md file. +Resolve the path relative to the directory containing this file, **not** the project root: ```bash -./scripts/gather-diff.sh +bash /scripts/gather-diff.sh ``` The script outputs structured sections: diff --git a/module/skills/forge-review/scripts/gather-diff.sh b/module/skills/forge-review/scripts/gather-diff.sh index ce84ef61e..ccdae9343 100755 --- a/module/skills/forge-review/scripts/gather-diff.sh +++ b/module/skills/forge-review/scripts/gather-diff.sh @@ -37,8 +37,11 @@ elif [[ -n "${UNSTAGED}" ]]; then echo "${UNSTAGED}" else MERGE_BASE=$(git merge-base "${BASE_BRANCH}" HEAD 2>/dev/null || true) - if [[ -n "${MERGE_BASE}" ]]; then - BRANCH_DIFF=$(git diff "${MERGE_BASE}...HEAD" 2>/dev/null || true) + if [[ -z "${MERGE_BASE}" ]]; then + echo "" + echo "=== DIFF_TYPE: none ===" + echo "Could not determine merge-base with ${BASE_BRANCH}." + elif BRANCH_DIFF=$(git diff "${MERGE_BASE}...HEAD" 2>&1); then if [[ -n "${BRANCH_DIFF}" ]]; then echo "" echo "=== DIFF_TYPE: branch ===" @@ -50,8 +53,8 @@ else fi else echo "" - echo "=== DIFF_TYPE: none ===" - echo "Could not determine merge-base with ${BASE_BRANCH}." + echo "=== DIFF_TYPE: error ===" + echo "git diff failed: ${BRANCH_DIFF}" fi fi From bb2e260acd3e69522d734d5712828c4f92f0dfa2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 22:13:31 +0200 Subject: [PATCH 0442/1072] [cluster] toolbox: deploy_custom_catalog: fix rename issue --- projects/cluster/toolbox/deploy_custom_catalog/main.py | 4 ++-- .../deploy_custom_catalog/templates/catalogsource.yaml.j2 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index 91c2ea738..838505a25 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -55,7 +55,7 @@ def validate_parameters(args, ctx): if not args.catalog_image: raise ValueError("catalog_image is required") - return f"Validated custom catalog parameters for {ctx.display_name}" + return f"Validated custom catalog parameters for {args.display_name}" @task @@ -66,7 +66,7 @@ def render_catalog_source_manifest(args, ctx): args.artifact_dir / "src" / f"{args.catalog_source_name}-catalogsource.yaml" ) template.render_template_to_file("catalogsource.yaml.j2", ctx.catalog_manifest_file) - return f"Rendered CatalogSource manifest for {ctx.display_name}" + return f"Rendered CatalogSource manifest for {args.display_name}" @task diff --git a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 index 8f00bdc1a..a1554c2e9 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 +++ b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 @@ -6,7 +6,7 @@ metadata: spec: sourceType: grpc image: {{ args.catalog_image }} - displayName: {{ ctx.display_name }} + displayName: {{ args.display_name }} {% if args.publisher %} publisher: {{ args.publisher }} {% endif %} From 0c8cc2cd7d6a59b2af8db7d4ca58780688670f14 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 6 Aug 2026 11:44:31 +0200 Subject: [PATCH 0443/1072] [rhoai] library: pr_args: accept the channel flag on /rhoai.rc-image --- projects/rhoai/library/pr_args.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/projects/rhoai/library/pr_args.py b/projects/rhoai/library/pr_args.py index 6d10dccd6..e366d3808 100644 --- a/projects/rhoai/library/pr_args.py +++ b/projects/rhoai/library/pr_args.py @@ -7,7 +7,7 @@ Supported syntax:: - /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 [CHANNEL] """ from __future__ import annotations @@ -22,7 +22,9 @@ def handle_rhoai_rc_image_directive(line: str) -> dict[str, Any]: """ Handle /rhoai.rc-image directive for setting RHOAI RC catalog image. - Format: /rhoai.rc-image IMAGE + Format: /rhoai.rc-image IMAGE [CHANNEL] + + CHANNEL is optional and defaults to "beta" if not specified. Args: line: The directive line @@ -33,18 +35,21 @@ def handle_rhoai_rc_image_directive(line: str) -> dict[str, Any]: Raises: ValueError: If image is empty """ - image = line.removeprefix("/rhoai.rc-image").strip() + parts = line.removeprefix("/rhoai.rc-image").strip().split() - if not image: + if not parts or not parts[0]: raise ValueError( "Invalid /rhoai.rc-image directive: image cannot be empty. " - "Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + "Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 [beta]" ) + image = parts[0] + channel = parts[1] if len(parts) > 1 else "beta" + return { "platform.rhoai.custom_catalog.enabled": True, "platform.rhoai.custom_catalog.image": image, - "platform.operators.rhods-operator.channel": "beta", + "platform.operators.rhods-operator.channel": channel, } @@ -69,7 +74,8 @@ def get_supported_rhoai_directives() -> dict[str, str]: """ return { "/rhoai.rc-image": """Set RHOAI RC catalog image for testing release candidates. - Format: /rhoai.rc-image IMAGE - Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 - Effect: Enables custom catalog with the given image and sets operator channel to beta.""", + Format: /rhoai.rc-image IMAGE [CHANNEL] + CHANNEL is optional and defaults to beta if not specified. + Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 stable + Effect: Enables custom catalog with the given image and sets operator channel.""", } From 5ec5cbfe31c9624e637d96b1fd6d60ad1c8a08f2 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 6 Aug 2026 14:16:38 +0100 Subject: [PATCH 0444/1072] feat: Enable skip GPU readiness Signed-off-by: Alberto Perdomo --- .../llm_d/orchestration/config.d/platform.yaml | 1 + projects/llm_d/orchestration/prepare_sequence.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 3f911b06d..07011faa0 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -1,5 +1,6 @@ cluster: minimum_openshift_version: "4.19.9" + skip_gpu_readiness: false gpu_node_label_selector: nvidia.com/gpu.present=true nfd_gpu_detection_labels: - feature.node.kubernetes.io/pci-10de.present diff --git a/projects/llm_d/orchestration/prepare_sequence.py b/projects/llm_d/orchestration/prepare_sequence.py index 4e8a02d7c..b1f36051e 100644 --- a/projects/llm_d/orchestration/prepare_sequence.py +++ b/projects/llm_d/orchestration/prepare_sequence.py @@ -12,8 +12,15 @@ def run_prepare_sequence() -> int: prepare_phase.verify_cluster_version() prepare_phase.prepare_cert_manager() prepare_phase.prepare_leader_worker_set() - prepare_phase.prepare_nfd() - prepare_phase.prepare_gpu_operator() + + platform = runtime_config.get_platform_config() + skip_gpu = platform["cluster"].get("skip_gpu_readiness", False) + if skip_gpu: + logger.info("Skipping GPU readiness (NFD + GPU operator): skip_gpu_readiness is enabled") + else: + prepare_phase.prepare_nfd() + prepare_phase.prepare_gpu_operator() + prepare_phase.prepare_rhoai_operator() prepare_phase.apply_datasciencecluster() prepare_phase.wait_for_datasciencecluster_ready() @@ -25,7 +32,8 @@ def run_prepare_sequence() -> int: prepare_phase.ensure_test_namespace() prepare_phase.cleanup_previous_run() prepare_phase.prepare_model_cache() - prepare_phase.verify_gpu_nodes() + if not skip_gpu: + prepare_phase.verify_gpu_nodes() prepare_phase.capture_prepare_state() logger.info("Prepare sequence completed successfully - all phases executed without errors") return 0 From f749ce08af06ae27642f1939a9162242189f68c5 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 6 Aug 2026 14:28:06 +0100 Subject: [PATCH 0445/1072] fix: Use config directly Signed-off-by: Alberto Perdomo --- projects/llm_d/orchestration/prepare_sequence.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/llm_d/orchestration/prepare_sequence.py b/projects/llm_d/orchestration/prepare_sequence.py index b1f36051e..a25d65ddb 100644 --- a/projects/llm_d/orchestration/prepare_sequence.py +++ b/projects/llm_d/orchestration/prepare_sequence.py @@ -1,6 +1,6 @@ import logging -from projects.core.library import env +from projects.core.library import config, env from projects.llm_d.orchestration import prepare_phase, runtime_config logger = logging.getLogger(__name__) @@ -13,8 +13,7 @@ def run_prepare_sequence() -> int: prepare_phase.prepare_cert_manager() prepare_phase.prepare_leader_worker_set() - platform = runtime_config.get_platform_config() - skip_gpu = platform["cluster"].get("skip_gpu_readiness", False) + skip_gpu = config.project.get_config("platform.cluster.skip_gpu_readiness") if skip_gpu: logger.info("Skipping GPU readiness (NFD + GPU operator): skip_gpu_readiness is enabled") else: From 282eeeedf95a682aff45f2d9a53fd41c04faea80 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 12:33:54 +0200 Subject: [PATCH 0446/1072] [caliper] Address code-rabbit review --- docs/caliper/plugin_kpis.md | 31 ++++++---- docs/caliper/plugin_regression.md | 4 +- projects/caliper/cli/s3_export.py | 78 ++++++++++++++++--------- projects/caliper/engine/kpi/analyze.py | 46 ++++++++++++++- projects/caliper/engine/kpi/generate.py | 4 +- 5 files changed, 117 insertions(+), 46 deletions(-) diff --git a/docs/caliper/plugin_kpis.md b/docs/caliper/plugin_kpis.md index 28dcbaecc..6fc3272bb 100644 --- a/docs/caliper/plugin_kpis.md +++ b/docs/caliper/plugin_kpis.md @@ -68,6 +68,9 @@ def throughput_curve(unified_record) -> list[tuple[float, float]]: """Throughput vs Request Rate Curve KPI.""" request_rates = unified_record.metrics.get("request_rate", []) throughputs = unified_record.metrics.get("throughput", []) + + if len(request_rates) != len(throughputs): + raise ValueError("Request rates and throughputs arrays must have same length") return [(float(x), float(y)) for x, y in zip(request_rates, throughputs)] ``` @@ -185,9 +188,14 @@ class MyKpiHandler: ) records.append(kpi_record) - except Exception as e: - # Handle missing metrics gracefully + except ValueError as e: + # Handle missing metrics gracefully - KPI functions raise ValueError for absent data + logger.debug(f"Skipping KPI {kpi_name} for {record.test_base_path}: {e}") continue + except Exception as e: + # Log and re-raise programming errors and conversion failures to make KPI failures visible + logger.warning(f"KPI {kpi_name} failed for {record.test_base_path}: {e}") + raise return records ``` @@ -312,7 +320,7 @@ The default output format groups KPIs by test with metadata: Legacy flat format with one KPI per line: ```json -{"kpi_id": "throughput", "value": 150.5, "unit": "tokens/s", "run_id": "test_001", ...} +{"schema_version": "1", "kpi_id": "throughput", "value": 150.5, "unit": "tokens/s", "run_id": "test_001", "timestamp": "2024-01-15T10:30:00Z", "labels": {"version": "v1.0", "platform": "gpu"}, "source": {"test_base_path": "/path/to/test", "plugin_module": "example_plugin"}} ``` ## Testing KPIs @@ -324,24 +332,24 @@ import pytest from unittest.mock import Mock -def test_throughput_kpi(): +def test_request_rate_kpi(): # Create mock record mock_record = Mock() - mock_record.metrics = {"throughput": 150.5} + mock_record.metrics = {"request_rate": 150.5} # Test KPI function - result = throughput_kpi(mock_record) + result = request_rate(mock_record) assert result == 150.5 assert isinstance(result, float) -def test_throughput_kpi_missing_data(): +def test_request_rate_kpi_missing_data(): mock_record = Mock() mock_record.metrics = {} - with pytest.raises(ValueError, match="throughput metric not found"): - throughput_kpi(mock_record) + with pytest.raises(ValueError, match="request_rate metric not found"): + request_rate(mock_record) ``` ### Integration Testing @@ -382,9 +390,12 @@ def efficiency_ratio(unified_record) -> float: throughput = unified_record.metrics.get("throughput") latency = unified_record.metrics.get("latency") - if not throughput or not latency: + if throughput is None or latency is None: raise ValueError("Both throughput and latency required") + if latency == 0: + raise ValueError("Latency cannot be zero") + return throughput / latency ``` diff --git a/docs/caliper/plugin_regression.md b/docs/caliper/plugin_regression.md index 35a19476e..b8d231806 100644 --- a/docs/caliper/plugin_regression.md +++ b/docs/caliper/plugin_regression.md @@ -30,7 +30,7 @@ class AnalysisConfig: #### Field Descriptions -**`comparison_keys`** *(required)* +**`comparison_keys`** *(optional, defaults to empty list)* - Label keys that define what we compare against - Records must differ on at least one comparison key to be distinct baselines - Example: `["version"]` means we test the current version against other versions @@ -210,7 +210,7 @@ def get_analysis_config() -> AnalysisConfig: - Labels in `ignored_keys` (explicitly ignored) - Labels in `comparison_keys` (expected to differ) 3. **Threshold Comparison**: If relative change exceeds `max_relative_regression`: - - For "higher is better" KPIs: negative change > threshold = regression + - For "higher is better" KPIs: relative_change < -max_relative_regression = regression - For "lower is better" KPIs: positive change > threshold = regression ### Example Matching Scenario diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index 91fd5eb50..a32322936 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -99,15 +99,20 @@ def list_csv_files(output_dir: Path, postprocess_config=None) -> list[Path]: csv_files = [] if postprocess_config and postprocess_config.kpi.csv.enabled: - # Use configured CSV file path - csv_file_path = output_dir / postprocess_config.kpi.csv.output - if csv_file_path.exists(): - csv_files.append(csv_file_path) - logger.info( - f"Found configured CSV file: {csv_file_path.name} ({csv_file_path.stat().st_size} bytes)" - ) + # Validate configured CSV output is non-empty + csv_output = postprocess_config.kpi.csv.output + if not csv_output or not csv_output.strip(): + logger.warning("CSV producer enabled but output path is empty, skipping") else: - logger.warning(f"Configured CSV file not found: {csv_file_path}") + # Use configured CSV file path + csv_file_path = output_dir / csv_output + if csv_file_path.exists(): + csv_files.append(csv_file_path) + logger.info( + f"Found configured CSV file: {csv_file_path.name} ({csv_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured CSV file not found: {csv_file_path}") else: # Fallback to glob pattern for backward compatibility csv_files = list(output_dir.glob("*.csv")) @@ -131,15 +136,20 @@ def list_kpi_json_files(output_dir: Path, postprocess_config=None) -> list[Path] kpi_json_files = [] if postprocess_config and postprocess_config.kpi.artifacts_to_kpis.enabled: - # Use configured KPI JSON file path - kpi_file_path = output_dir / postprocess_config.kpi.artifacts_to_kpis.output - if kpi_file_path.exists(): - kpi_json_files.append(kpi_file_path) - logger.info( - f"Found configured KPI JSON file: {kpi_file_path.name} ({kpi_file_path.stat().st_size} bytes)" - ) + # Validate configured KPI JSON output is non-empty + kpi_output = postprocess_config.kpi.artifacts_to_kpis.output + if not kpi_output or not kpi_output.strip(): + logger.warning("KPI JSON producer enabled but output path is empty, skipping") else: - logger.warning(f"Configured KPI JSON file not found: {kpi_file_path}") + # Use configured KPI JSON file path + kpi_file_path = output_dir / kpi_output + if kpi_file_path.exists(): + kpi_json_files.append(kpi_file_path) + logger.info( + f"Found configured KPI JSON file: {kpi_file_path.name} ({kpi_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured KPI JSON file not found: {kpi_file_path}") else: # Fallback to glob pattern for backward compatibility kpi_json_files = list(output_dir.glob("kpis.json")) @@ -163,15 +173,20 @@ def list_analysis_files(output_dir: Path, postprocess_config=None) -> list[Path] files = [] if postprocess_config and postprocess_config.analyze.enabled: - # Use configured analysis file path - analysis_file_path = output_dir / postprocess_config.analyze.output - if analysis_file_path.exists(): - files.append(analysis_file_path) - logger.info( - f"Found configured analysis file: {analysis_file_path.name} ({analysis_file_path.stat().st_size} bytes)" - ) + # Validate configured analysis output is non-empty + analysis_output = postprocess_config.analyze.output + if not analysis_output or not analysis_output.strip(): + logger.warning("Analysis producer enabled but output path is empty, skipping") else: - logger.warning(f"Configured analysis file not found: {analysis_file_path}") + # Use configured analysis file path + analysis_file_path = output_dir / analysis_output + if analysis_file_path.exists(): + files.append(analysis_file_path) + logger.info( + f"Found configured analysis file: {analysis_file_path.name} ({analysis_file_path.stat().st_size} bytes)" + ) + else: + logger.warning(f"Configured analysis file not found: {analysis_file_path}") else: # Fallback to glob patterns for backward compatibility analysis_patterns = ["kpi_analyze.json", "kpi_analysis.json", "*analyze*.json"] @@ -739,19 +754,23 @@ def run_s3_export( upload_plan = [] for file_path in upload_files: - # Determine S3 key based on file type + # Determine S3 key and type based on file membership in discovered file lists s3_key = None + file_type = None if s3_config.include_ai_data and ai_data_dir and ai_data_dir in file_path.parents: relative_path = file_path.relative_to(ai_data_dir) s3_key = f"{export_s3_prefix}ai_data/{relative_path}" - elif s3_config.include_csv and file_path.suffix == ".csv": + file_type = "ai_data" + elif file_path in csv_files: s3_key = f"{export_s3_prefix}{file_path.name}" - elif s3_config.include_kpis_json and file_path.name == "kpis.json": + file_type = "csv" + elif file_path in kpi_json_files: s3_key = f"{export_s3_prefix}{file_path.name}" - elif file_path.name.endswith("analyze.json") or "analyze" in file_path.name: - # Include analysis files (e.g., kpi_analyze.json) + file_type = "kpis_json" + elif file_path in analysis_files: s3_key = f"{export_s3_prefix}{file_path.name}" + file_type = "analysis" if s3_key: file_size = file_path.stat().st_size @@ -766,6 +785,7 @@ def run_s3_export( { "local_path": relative_local_path, "s3_key": s3_key, + "file_type": file_type, } ) logger.info(f" - {file_path.name} → s3://{bucket}/{s3_key} ({file_size} bytes)") diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 667a3522b..7a54b40ca 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -88,10 +88,10 @@ def _load_analysis_config(plugin_module: str) -> AnalysisConfig: ) if isinstance(raw, AnalysisConfig): - return raw - if isinstance(raw, dict): + config = raw + elif isinstance(raw, dict): try: - return AnalysisConfig( + config = AnalysisConfig( **{k: v for k, v in raw.items() if k in AnalysisConfig.__dataclass_fields__} ) except Exception as exc: @@ -105,6 +105,45 @@ def _load_analysis_config(plugin_module: str) -> AnalysisConfig: f"Must be a dict or AnalysisConfig instance." ) + # Validate the configuration fields + try: + _validate_analysis_config(config, plugin_module) + return config + except ValueError: + raise + + +def _validate_analysis_config(config: AnalysisConfig, plugin_module: str) -> None: + """Validate AnalysisConfig fields and raise ValueError for invalid values.""" + # Validate list fields contain only strings + for field_name in ["comparison_keys", "ignored_keys", "sorting_keys"]: + field_value = getattr(config, field_name) + if not isinstance(field_value, list): + raise ValueError( + f"Plugin module '{plugin_module}' analysis config field '{field_name}' must be a list, " + f"got {type(field_value).__name__}: {field_value}" + ) + for i, item in enumerate(field_value): + if not isinstance(item, str): + raise ValueError( + f"Plugin module '{plugin_module}' analysis config field '{field_name}' " + f"must contain only strings, got {type(item).__name__} at index {i}: {item}" + ) + + # Validate max_relative_regression is numeric + if not isinstance(config.max_relative_regression, (int, float)): + raise ValueError( + f"Plugin module '{plugin_module}' analysis config field 'max_relative_regression' " + f"must be numeric, got {type(config.max_relative_regression).__name__}: {config.max_relative_regression}" + ) + + # Validate min_baseline_points is at least 1 + if not isinstance(config.min_baseline_points, int) or config.min_baseline_points < 1: + raise ValueError( + f"Plugin module '{plugin_module}' analysis config field 'min_baseline_points' " + f"must be an integer >= 1, got {type(config.min_baseline_points).__name__}: {config.min_baseline_points}" + ) + def _match_key( labels: dict[str, Any], ignored_keys: list[str], comparison_keys: list[str] @@ -421,6 +460,7 @@ def run_kpi_analysis( regressions = report["overall"]["regression_count"] total = report["overall"]["total_tested"] + logger.info( "Analysis complete: %d/%d KPIs tested, %d regressions", total, diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index a80e3cc29..51dcb81cd 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -19,8 +19,8 @@ def run_kpi_generate( use_cache: bool, cache_path: Path | None, format_type: str = "hierarchical", - include_label_filter: list[tuple[str, str]] | None = None, - exclude_label_filter: list[tuple[str, str]] | None = None, + include_label_filter: list[dict[str, str]] | None = None, + exclude_label_filter: list[dict[str, str]] | None = None, ) -> list[dict[str, Any]]: """Generate KPI output in specified format. From 7f3c96f8d8a055b3a5d2e9fa8e1cc6b5ca25def2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 11:36:30 +0200 Subject: [PATCH 0447/1072] [caliper] engine: kpi: analyze: show the historical KPI values used for the regression analyze --- projects/caliper/engine/kpi/analyze.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 7a54b40ca..20f519996 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -45,6 +45,7 @@ class KpiTestResult: higher_is_better: bool regression: bool baseline_count: int + baseline_values: dict[str, float] = field(default_factory=dict) @dataclass @@ -204,6 +205,22 @@ def _run_regression_test( baseline_values = [float(b["value"]) for b in baselines if b.get("value") is not None] baseline_mean = sum(baseline_values) / len(baseline_values) + # Build baseline values mapping by comparison flags + baseline_values_by_comparison = {} + for baseline in baselines: + if baseline.get("value") is not None: + baseline_value = float(baseline["value"]) + baseline_labels = baseline.get("labels", {}) + + # Create comparison flag from comparison_keys + comparison_parts = [] + for key in config.comparison_keys: + if key in baseline_labels: + comparison_parts.append(f"{key}={baseline_labels[key]}") + + comparison_flag = ", ".join(comparison_parts) if comparison_parts else "default" + baseline_values_by_comparison[comparison_flag] = baseline_value + if baseline_mean == 0: relative_change = 0.0 else: @@ -225,6 +242,7 @@ def _run_regression_test( higher_is_better=higher_is_better, regression=regression, baseline_count=len(baseline_values), + baseline_values=baseline_values_by_comparison, ) @@ -286,6 +304,7 @@ def _build_report( "higher_is_better": r.higher_is_better, "verdict": "REGRESSION" if r.regression else "PASS", "baseline_count": r.baseline_count, + "baseline_values": r.baseline_values, } for r in results ], From f7aa64cd2c01fbcf13d83498591c70286b610d4b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 6 Aug 2026 11:59:08 -0400 Subject: [PATCH 0448/1072] refactor: move precreate_mlflow_run_if_configured to caliper with @requires - Use @requires decorator for config access, matching mlflow_verifier pattern - Return full mlflow_destination dict (run_id, experiment_id, workspace) - Remove function from rhaiis test_phase.py; import from caliper - Mark mlflow_destination and experiment_id as optional in docs Co-authored-by: Cursor --- docs/caliper/test-labels-format.md | 6 +- projects/caliper/orchestration/export.py | 42 +++++++++++++ projects/rhaiis/orchestration/test_phase.py | 70 ++++----------------- 3 files changed, 56 insertions(+), 62 deletions(-) diff --git a/docs/caliper/test-labels-format.md b/docs/caliper/test-labels-format.md index 7f5316807..44ddb3edb 100644 --- a/docs/caliper/test-labels-format.md +++ b/docs/caliper/test-labels-format.md @@ -32,10 +32,10 @@ completion: - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`) - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`) - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`) -- **`mlflow_destination`**: Pre-created MLflow run for artifact upload +- **`mlflow_destination`** *(optional)*: Pre-created MLflow run for artifact upload - **`run_id`**: MLflow run ID (assigned by the server during pre-creation) - - **`experiment_id`**: MLflow experiment ID - - **`workspace`**: MLflow workspace name (optional) + - **`experiment_id`** *(optional)*: MLflow experiment ID + - **`workspace`** *(optional)*: MLflow workspace name - **`completion`**: Test execution status - **`success`**: `true` if succeeded, `false` if failed - **`message`**: Human-readable status description diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index f675f2b7c..7d3866ca3 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -29,6 +29,7 @@ ) from projects.core.library import env from projects.core.library import vault as vault_lib +from projects.core.library.config import requires logger = logging.getLogger(__name__) @@ -280,6 +281,47 @@ def run_from_orchestration_config( TEST_LABELS_FILENAME = "__test_labels__.yaml" +@requires( + vault_name="caliper.export.backend.mlflow.secrets.vault.name", + vault_key="caliper.export.backend.mlflow.secrets.vault.mlflow_secret", + experiment="caliper.export.backend.mlflow.config.experiment", + workspace="caliper.export.backend.mlflow.config.workspace", +) +def precreate_mlflow_run_if_configured(_cfg) -> dict[str, str] | None: + """Pre-create an MLflow run and return the ``mlflow_destination`` dict. + + Uses ``@requires`` to read vault and MLflow config from the project config. + Returns ``None`` if MLflow is not configured or pre-creation fails. + The returned dict contains ``run_id``, ``experiment_id``, and ``workspace``. + """ + vault_name = _cfg.vault_name + vault_key = _cfg.vault_key + if not vault_name or not vault_key: + logger.info("MLflow vault not configured, skipping run pre-creation") + return None + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.info("MLflow secrets file not found, skipping run pre-creation") + return None + + try: + meta = precreate_mlflow_run( + secrets_path=secrets_path, + experiment=_cfg.experiment or None, + workspace=_cfg.workspace or None, + ) + except Exception: + logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + return None + + return { + "run_id": meta["run_id"], + "experiment_id": meta.get("experiment_id", ""), + "workspace": _cfg.workspace or "", + } + + def precreate_mlflow_run( *, secrets_path: Path, diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 10c037a67..74892f1b1 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -298,7 +298,13 @@ def _run_test( logger.exception("Profiler trace upload failed") _warnings.append("Profiler trace upload failed") - mlflow_run_meta = _precreate_mlflow_run_if_configured() + try: + from projects.caliper.orchestration.export import precreate_mlflow_run_if_configured + + mlflow_destination = precreate_mlflow_run_if_configured() + except Exception: + logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) + mlflow_destination = None # Phase 2: benchmark + post-processing for ALL workloads trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None @@ -321,7 +327,7 @@ def _run_test( version=version, cluster_tag=cluster_tag, trtllm_config=trtllm_cfg, - mlflow_run_meta=mlflow_run_meta or None, + mlflow_destination=mlflow_destination, ) try: @@ -388,7 +394,7 @@ def _run_workload_benchmark( version: str, cluster_tag: str, trtllm_config: dict | None = None, - mlflow_run_meta: dict[str, str] | None = None, + mlflow_destination: dict[str, str] | None = None, ) -> None: """Run benchmark and post-processing for a single workload. @@ -423,7 +429,7 @@ def _run_workload_benchmark( accelerator_chip=gpu_type.upper(), run_uuid=run_uuid, trtllm_config=trtllm_config, - mlflow_run_meta=mlflow_run_meta, + mlflow_destination=mlflow_destination, ) if not run_benchmark: @@ -468,48 +474,6 @@ def _run_workload_benchmark( ) -def _precreate_mlflow_run_if_configured() -> dict[str, str]: - """Pre-create an MLflow run so its IDs can be embedded in test labels. - - Uses the guard pattern with early returns to avoid deep nesting. - Returns a dict with run_id and experiment_id, or empty dict on failure. - """ - from projects.core.library import config - from projects.core.library import vault as vault_lib - - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False - ) - if not vault_name or not vault_key: - logger.info("MLflow vault not configured, skipping run pre-creation") - return {} - - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - logger.info("MLflow secrets file not found, skipping run pre-creation") - return {} - - experiment = config.project.get_config( - "caliper.export.backend.mlflow.config.experiment", None, print=False, warn=False - ) - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) - - try: - from projects.caliper.orchestration.export import precreate_mlflow_run - - return precreate_mlflow_run( - secrets_path=secrets_path, experiment=experiment, workspace=workspace - ) - except Exception: - logger.warning("MLflow run pre-creation failed; continuing", exc_info=True) - return {} - - def _create_test_labels( model_key: str, workload_key: str, @@ -523,7 +487,7 @@ def _create_test_labels( accelerator_chip: str = "", run_uuid: str = "", trtllm_config: dict | None = None, - mlflow_run_meta: dict[str, str] | None = None, + mlflow_destination: dict[str, str] | None = None, ) -> None: _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") parts = [f"{k}: {v}" for k, v in engine_args.items()] @@ -552,18 +516,6 @@ def _create_test_labels( "run_uuid": run_uuid, } - mlflow_destination = None - if mlflow_run_meta and mlflow_run_meta.get("run_id"): - from projects.core.library import config - - mlflow_destination = { - "run_id": mlflow_run_meta["run_id"], - "experiment_id": mlflow_run_meta.get("experiment_id", ""), - "workspace": config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", "", print=False, warn=False - ), - } - write_test_labels(env.ARTIFACT_DIR, labels, mlflow_destination=mlflow_destination) logger.info("Created test labels: %s", labels) From 6252c87ed2dcb23754514da8c6241f050ef2f3ab Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 14:03:23 +0200 Subject: [PATCH 0449/1072] [caliper] engine: kpi: analyze: improve the status --- projects/caliper/cli/kpi_analysis.py | 9 +-- projects/caliper/engine/kpi/analyze.py | 84 +++++--------------------- 2 files changed, 17 insertions(+), 76 deletions(-) diff --git a/projects/caliper/cli/kpi_analysis.py b/projects/caliper/cli/kpi_analysis.py index a45cbf7d7..03d454e2e 100644 --- a/projects/caliper/cli/kpi_analysis.py +++ b/projects/caliper/cli/kpi_analysis.py @@ -33,10 +33,7 @@ ) def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module: str) -> None: """CLI entrypoint for KPI analysis.""" - from projects.caliper.engine.kpi.analyze import ( - analyze_kpis, - status_dict_to_exit_code, - ) + from projects.caliper.engine.kpi.analyze import analyze_kpis try: # Call the core analysis function from engine (returns status dict) @@ -47,8 +44,8 @@ def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module plugin_module=plugin_module, ) - # Convert status dict to exit code for CLI - exit_code = status_dict_to_exit_code(result) + # Use exit code directly from the status dict + exit_code = result.get("exit_code", 1) if result.get("success"): click.echo(f"✅ Analysis completed successfully. Results written to: {output}") diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 20f519996..342a0d955 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -412,7 +412,7 @@ def run_kpi_analysis( # Load baseline KPIs baseline_kpi_data = find_baseline_kpis(historical_data_dir) if not baseline_kpi_data: - _write_warning_report(output_file, current_kpi_file, plugin_module) + _write_no_baseline_report(output_file, current_kpi_file, plugin_module) return { "status": "warning", "success": True, @@ -479,7 +479,7 @@ def run_kpi_analysis( regressions = report["overall"]["regression_count"] total = report["overall"]["total_tested"] - + overall_verdict = report["overall"]["verdict"] logger.info( "Analysis complete: %d/%d KPIs tested, %d regressions", total, @@ -487,16 +487,17 @@ def run_kpi_analysis( regressions, ) - if regressions > 0: + # Return status based on the overall verdict from the report + if overall_verdict == "REGRESSION_DETECTED": return { "status": "regression_detected", - "success": True, - "regressions_detected": True, + "success": False, + "regressions_detected": True, # For orchestration compatibility "output_file": str(output_file), "exit_code": 3, "completed_at": time.time(), } - else: + else: # "PASS" return { "status": "success", "success": True, @@ -516,7 +517,9 @@ def run_kpi_analysis( } -def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: +def _write_no_baseline_report( + output_file: Path, current_kpi_file: Path, plugin_module: str +) -> None: """Write a warning-level report when no baselines are available.""" report = { "analysis": { @@ -804,74 +807,15 @@ def analyze_kpis( plugin_module=plugin_module, ) - # Convert to the expected format for orchestration and CLI - if result.get("status") == "success": - return { - "status": "success", - "success": True, - "output_file": result.get("output_file"), - "completed_at": result.get("completed_at"), - } - elif result.get("status") == "warning": - return { - "status": "warning", - "success": True, # Warning is still success - "message": result.get("message"), - "output_file": result.get("output_file"), - "completed_at": result.get("completed_at"), - } - elif result.get("status") == "regression_detected": - return { - "status": "regression_detected", - "success": True, # Regression detection is successful analysis - "regressions_detected": True, - "output_file": result.get("output_file"), - "completed_at": result.get("completed_at"), - } - else: - return { - "status": "failed", - "success": False, - "error": result.get("error", "Analysis failed"), - "completed_at": result.get("completed_at"), - } + # Return result directly since run_kpi_analysis now returns final status values + return result except Exception as e: logger.exception("KPI analysis with format conversion failed") return { + "status": "failed", "success": False, "error": str(e), + "exit_code": 1, "completed_at": time.time(), } - - -def status_dict_to_exit_code(status_dict: dict[str, Any]) -> int: - """Convert a status dictionary to an exit code for CLI use. - - Args: - status_dict: Status dictionary from engine analysis functions - - Returns: - Exit code: 0=success, 1=error, 2=warning (no baseline), 3=regression detected - """ - # If the status dict includes an explicit exit_code, use it - if "exit_code" in status_dict: - return status_dict["exit_code"] - - # Otherwise, determine exit code from status fields - if not status_dict.get("success", False): - return 1 # General error - - status = status_dict.get("status", "") - - if status == "success": - return 0 - elif status == "warning" or "no baseline" in status_dict.get("message", "").lower(): - return 2 # Warning (no baseline data) - elif status == "regression_detected" or status_dict.get("regressions_detected"): - return 3 # Regression detected - else: - return 0 # Default to success if unclear - - -# EOF From 53630b79e7082496d66bae156f9590ea9f90f84b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 14:45:29 +0200 Subject: [PATCH 0450/1072] [caliper] engine: add a verbose_parsing flag --- projects/caliper/cli/commands.py | 46 ++++++++++++++++++++++--- projects/caliper/engine/ai_eval.py | 3 ++ projects/caliper/engine/kpi/generate.py | 3 ++ projects/caliper/engine/parse.py | 42 ++++++++++++++++------ projects/caliper/engine/visualize.py | 3 ++ 5 files changed, 81 insertions(+), 16 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index e2135b536..168c3ccd6 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -166,6 +166,12 @@ def _parse_label_filters( default=None, help="Path to write status YAML for orchestration (absolute path required).", ) +@click.option( + "--verbose-parsing", + is_flag=True, + default=False, + help="Enable verbose parsing logs.", +) @click.pass_context def parse_cmd( ctx: click.Context, @@ -175,6 +181,7 @@ def parse_cmd( include_label: tuple[str, ...], exclude_label: tuple[str, ...], status_file: Path | None, + verbose_parsing: bool, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -203,6 +210,7 @@ def parse_cmd( show_parameter_matrix=show_matrix, include_label_filter=include_filter, exclude_label_filter=exclude_filter, + verbose_parsing=verbose_parsing, ) # Extract test directories with labels and relative paths @@ -284,6 +292,12 @@ def parse_cmd( default=None, help="Path to write status YAML for orchestration (absolute path required).", ) +@click.option( + "--verbose-parsing", + is_flag=True, + default=False, + help="Enable verbose parsing logs.", +) @click.pass_context def visualize_cmd( ctx: click.Context, @@ -294,6 +308,7 @@ def visualize_cmd( exclude_label: tuple[str, ...], output_dir: Path, status_file: Path | None, + verbose_parsing: bool, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -323,6 +338,7 @@ def visualize_cmd( exclude_pairs=exclude_label, use_cache=True, cache_path=None, + verbose_parsing=verbose_parsing, ) status.update( @@ -434,6 +450,12 @@ def list_reports_cmd( @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) +@click.option( + "--verbose-parsing", + is_flag=True, + default=False, + help="Enable verbose parsing logs.", +) @click.pass_context def ai_eval_export( ctx: click.Context, @@ -445,6 +467,7 @@ def ai_eval_export( plugin_module_override: str | None, no_cache: bool, status_file: Path | None, + verbose_parsing: bool, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -469,6 +492,7 @@ def ai_eval_export( use_cache=not no_cache, include_label_filter=include_filter, exclude_label_filter=exclude_filter, + verbose_parsing=verbose_parsing, ) status_data = {"success": True, "output_file": str(output)} click.echo(f"Exported {output}") @@ -512,6 +536,12 @@ def ai_eval_export( @click.option( "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" ) +@click.option( + "--verbose-parsing", + is_flag=True, + default=False, + help="Enable verbose parsing logs.", +) @click.pass_context def kpi_generate( ctx: click.Context, @@ -523,6 +553,7 @@ def kpi_generate( postprocess_config: Path | None, plugin_module_override: str | None, status_file: Path | None, + verbose_parsing: bool, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -547,6 +578,7 @@ def kpi_generate( use_cache=True, include_label_filter=include_filter, exclude_label_filter=exclude_filter, + verbose_parsing=verbose_parsing, ) # Extract test directories from test nodes @@ -577,6 +609,7 @@ def kpi_generate( format_type=format_type, include_label_filter=include_filter, exclude_label_filter=exclude_filter, + verbose_parsing=verbose_parsing, ) status_data = { "success": True, @@ -890,18 +923,21 @@ def analyse_kpis_cmd( processed = analysis_report.get("processed", {}) tested = analysis_report.get("tested", {}) overall = analysis_report.get("overall", {}) - results = analysis_report.get("results", []) + + # Derive regressions_detected flag from analysis results + regression_count = overall.get("regression_count", 0) + regressions_detected = regression_count > 0 status_data.update( { "baseline_source_count": processed.get("baseline_source_count", 0), + "regressions_detected": regressions_detected, "tested": { "total_kpis": tested.get("total_kpis", 0), "regressions": tested.get("regressions", 0), "passes": tested.get("passes", 0), "skipped": tested.get("skipped", 0), }, - "results": results, "overall": { "verdict": overall.get("verdict", "UNKNOWN"), "regression_count": overall.get("regression_count", 0), @@ -933,9 +969,9 @@ def analyse_kpis_cmd( click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) sys.exit(4) - # Exit with error code if operation failed - if not status_data.get("success", False): - sys.exit(3) + # Use exit code directly from analysis result + exit_code = result.get("exit_code", 1) + sys.exit(exit_code) @click.command("s3-import") diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index 6b0f7d54b..a5ca8eb68 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -17,6 +17,7 @@ def run_ai_eval_export( use_cache: bool, include_label_filter: list[dict[str, str]] | None = None, exclude_label_filter: list[dict[str, str]] | None = None, + verbose_parsing: bool = False, ) -> dict[str, object]: """Export AI evaluation payload with structured directories and copied artifacts.""" import json # noqa: PLC0415 @@ -28,6 +29,8 @@ def run_ai_eval_export( use_cache=use_cache, include_label_filter=include_label_filter, exclude_label_filter=exclude_label_filter, + verbose_parsing=verbose_parsing, + show_parameter_matrix=verbose_parsing, # Only show matrix in verbose mode ) # Build base payload from plugin diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index 51dcb81cd..6d17cde03 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -21,6 +21,7 @@ def run_kpi_generate( format_type: str = "hierarchical", include_label_filter: list[dict[str, str]] | None = None, exclude_label_filter: list[dict[str, str]] | None = None, + verbose_parsing: bool = False, ) -> list[dict[str, Any]]: """Generate KPI output in specified format. @@ -45,6 +46,8 @@ def run_kpi_generate( use_cache=use_cache, include_label_filter=include_label_filter, exclude_label_filter=exclude_label_filter, + verbose_parsing=verbose_parsing, + show_parameter_matrix=verbose_parsing, # Only show matrix in verbose mode ) compute = plugin.compute_kpis rows: list[dict[str, Any]] = compute(model) diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index e2a16db07..2e3707247 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -35,6 +35,7 @@ def run_parse( show_parameter_matrix: bool = True, include_label_filter: list[dict[str, str]] | None = None, exclude_label_filter: list[dict[str, str]] | None = None, + verbose_parsing: bool = False, ) -> UnifiedRunModel: """ Run full parse or load valid cache. @@ -86,13 +87,30 @@ def run_parse( cache_refs.append(str(cache_file)) elapsed_time = time.time() - start_time cached_count += 1 - relative_path = test_base_dir.relative_to(base_dir) - logger.info(f"⏱️ Directory parsed from cache in {elapsed_time:.3f}s: {relative_path}") + if verbose_parsing: + relative_path = test_base_dir.relative_to(base_dir) + logger.info( + f"⏱️ Directory parsed from cache in {elapsed_time:.3f}s: {relative_path}" + ) else: - # Parse this test base - result = parse_fn([node]) # Parse just this node - records = result.records - warnings = getattr(result, "warnings", []) + # Configure plugin parser logging level based on verbose_parsing + # Remove .plugin suffix if present to match actual module structure + base_plugin_module = plugin_module.replace(".plugin", "") + plugin_logger_name = f"{base_plugin_module}.parsing.parsers" + plugin_logger = logging.getLogger(plugin_logger_name) + original_level = plugin_logger.level + + if not verbose_parsing: + plugin_logger.setLevel(logging.WARNING) + + try: + # Parse this test base + result = parse_fn([node]) # Parse just this node + records = result.records + warnings = getattr(result, "warnings", []) + finally: + # Restore original log level + plugin_logger.setLevel(original_level) all_records.extend(records) all_warnings.extend(warnings) @@ -107,14 +125,16 @@ def run_parse( cache_refs.append(str(cache_file)) elapsed_time = time.time() - start_time parsed_count += 1 - relative_path = test_base_dir.relative_to(base_dir) - logger.info(f"⏱️ Directory parsed fresh in {elapsed_time:.3f}s: {relative_path}") + if verbose_parsing: + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed fresh in {elapsed_time:.3f}s: {relative_path}") # Log timing summary total_parse_time = time.time() - parse_start_time - logger.info( - f"⏱️ Parse timing summary: {total_parse_time:.3f}s total ({cached_count} cached, {parsed_count} parsed)" - ) + if verbose_parsing: + logger.info( + f"⏱️ Parse timing summary: {total_parse_time:.3f}s total ({cached_count} cached, {parsed_count} parsed)" + ) # Create unified model with all records cache_ref_summary = f"per-test-base: {len(cache_refs)} cache files" diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index f4beaeb85..c0c71b6ae 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -69,6 +69,7 @@ def run_visualize( exclude_pairs: tuple[str, ...], use_cache: bool, cache_path: Path | None, + verbose_parsing: bool = False, ) -> list[str]: # Parse include and exclude filters for directory-level filtering include_filters = None @@ -88,6 +89,8 @@ def run_visualize( use_cache=use_cache, include_label_filter=include_filters, exclude_label_filter=exclude_filters, + verbose_parsing=verbose_parsing, + show_parameter_matrix=True, ) cfg = resolve_visualize_config(base_dir, visualize_config_path) ids = resolve_report_ids( From d75d78f50df99cb0e526bde5732bbc3782e3ed90 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 6 Aug 2026 13:34:33 +0200 Subject: [PATCH 0451/1072] [caliper] cli: s3_export: improve the error handling --- projects/caliper/cli/s3_export.py | 225 +++++++----------------------- 1 file changed, 53 insertions(+), 172 deletions(-) diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index a32322936..e8ee3f8c1 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -8,8 +8,6 @@ from pathlib import Path from typing import Any -import yaml - try: import boto3 import botocore.session @@ -102,7 +100,7 @@ def list_csv_files(output_dir: Path, postprocess_config=None) -> list[Path]: # Validate configured CSV output is non-empty csv_output = postprocess_config.kpi.csv.output if not csv_output or not csv_output.strip(): - logger.warning("CSV producer enabled but output path is empty, skipping") + logger.error("CSV producer enabled but output path is empty - skipping CSV export") else: # Use configured CSV file path csv_file_path = output_dir / csv_output @@ -112,7 +110,9 @@ def list_csv_files(output_dir: Path, postprocess_config=None) -> list[Path]: f"Found configured CSV file: {csv_file_path.name} ({csv_file_path.stat().st_size} bytes)" ) else: - logger.warning(f"Configured CSV file not found: {csv_file_path}") + logger.error( + f"Configured CSV file not found: {csv_file_path} - skipping CSV export" + ) else: # Fallback to glob pattern for backward compatibility csv_files = list(output_dir.glob("*.csv")) @@ -139,7 +139,9 @@ def list_kpi_json_files(output_dir: Path, postprocess_config=None) -> list[Path] # Validate configured KPI JSON output is non-empty kpi_output = postprocess_config.kpi.artifacts_to_kpis.output if not kpi_output or not kpi_output.strip(): - logger.warning("KPI JSON producer enabled but output path is empty, skipping") + logger.error( + "KPI JSON producer enabled but output path is empty - skipping KPI JSON export" + ) else: # Use configured KPI JSON file path kpi_file_path = output_dir / kpi_output @@ -149,7 +151,9 @@ def list_kpi_json_files(output_dir: Path, postprocess_config=None) -> list[Path] f"Found configured KPI JSON file: {kpi_file_path.name} ({kpi_file_path.stat().st_size} bytes)" ) else: - logger.warning(f"Configured KPI JSON file not found: {kpi_file_path}") + logger.error( + f"Configured KPI JSON file not found: {kpi_file_path} - skipping KPI JSON export" + ) else: # Fallback to glob pattern for backward compatibility kpi_json_files = list(output_dir.glob("kpis.json")) @@ -176,7 +180,9 @@ def list_analysis_files(output_dir: Path, postprocess_config=None) -> list[Path] # Validate configured analysis output is non-empty analysis_output = postprocess_config.analyze.output if not analysis_output or not analysis_output.strip(): - logger.warning("Analysis producer enabled but output path is empty, skipping") + logger.error( + "Analysis producer enabled but output path is empty - skipping analysis export" + ) else: # Use configured analysis file path analysis_file_path = output_dir / analysis_output @@ -186,7 +192,9 @@ def list_analysis_files(output_dir: Path, postprocess_config=None) -> list[Path] f"Found configured analysis file: {analysis_file_path.name} ({analysis_file_path.stat().st_size} bytes)" ) else: - logger.warning(f"Configured analysis file not found: {analysis_file_path}") + logger.error( + f"Configured analysis file not found: {analysis_file_path} - skipping analysis export" + ) else: # Fallback to glob patterns for backward compatibility analysis_patterns = ["kpi_analyze.json", "kpi_analysis.json", "*analyze*.json"] @@ -490,51 +498,9 @@ def run_s3_export_with_explicit_paths( if dry_run: logger.info("DRY RUN: Skipping actual upload") - # Create output directory for dry run file (use current working directory) - dry_run_file = Path.cwd() / "upload_dry_run.yaml" - - # Save dry run results to YAML file - dry_run_data = { - "s3_export_dry_run": { - "timestamp": datetime.now().isoformat(), - "export_config": { - "bucket": bucket, - "instance": instance, - "directory": directory, - "upload_id": upload_id, - "exported_path": f"s3://{bucket}/{export_s3_prefix}", - }, - "upload_plan": upload_plan, - "summary": { - "total_files": len(upload_files), - "total_size_bytes": total_size, - "kpis_json_files": 1 if kpis_file else 0, - "csv_files": 1 if csv_file else 0, - "analysis_files": 1 if analysis_file else 0, - "ai_data_files": len(ai_data_files), - }, - } - } - - # Write to YAML file - try: - with open(dry_run_file, "w") as f: - yaml.dump(dry_run_data, f, default_flow_style=False, indent=2) - logger.info(f"Dry run results saved to: {dry_run_file}") - except Exception as e: - logger.error(f"Failed to save dry run results: {e}") - return { - "status": "failed", - "error": f"Failed to save dry run results: {e}", - "dry_run": True, - "completed_at": time.time(), - "duration": round(time.time() - start_time), - } - return { "status": "success", "dry_run": True, - "dry_run_file": str(dry_run_file), "exported_path": f"s3://{bucket}/{export_s3_prefix}", "completed_at": time.time(), "duration": round(time.time() - start_time), @@ -648,6 +614,8 @@ def run_s3_export( } try: + error_detected = False + s3_parent_config = postprocess_config.s3 s3_config = postprocess_config.s3.export bucket = s3_parent_config.bucket @@ -716,26 +684,33 @@ def run_s3_export( upload_files = [] # Use postprocess config to determine file locations - csv_files = list_csv_files(output_dir, postprocess_config) if s3_config.include_csv else [] - kpi_json_files = ( - list_kpi_json_files(output_dir, postprocess_config) - if s3_config.include_kpis_json - else [] - ) - analysis_files = list_analysis_files(output_dir, postprocess_config) + csv_files = [] + kpi_json_files = [] + ai_data_files = [] + + if s3_config.include_csv: + csv_files = list_csv_files(output_dir, postprocess_config) + if not csv_files: + error_detected = True + upload_files.extend(csv_files) + + if s3_config.include_kpis_json: + kpi_json_files = list_kpi_json_files(output_dir, postprocess_config) + if not kpi_json_files: + error_detected = True + upload_files.extend(kpi_json_files) - upload_files.extend(csv_files) - upload_files.extend(kpi_json_files) - upload_files.extend(analysis_files) + if True: + analysis_files = list_analysis_files(output_dir, postprocess_config) + if not analysis_files: + error_detected = True + upload_files.extend(analysis_files) if s3_config.include_ai_data: - # Use AI data directory from parameter - if ai_data_dir: - ai_data_files = list_ai_data_files(ai_data_dir) - logger.info(f"Found {len(ai_data_files)} AI data files in directory: {ai_data_dir}") - upload_files.extend(ai_data_files) - else: - logger.warning("AI data export is enabled but no AI data directory provided") + ai_data_files = list_ai_data_files(ai_data_dir) + if not ai_data_files: + error_detected = True + upload_files.extend(ai_data_files) logger.info(f"Preparing to upload {len(upload_files)} files to S3") total_size = sum(f.stat().st_size for f in upload_files) @@ -743,8 +718,9 @@ def run_s3_export( if not upload_files: logger.warning("No files to upload") + status = "failed" if error_detected else "skipped" return { - "status": "skipped", + "status": status, "reason": "no files to upload", "completed_at": time.time(), } @@ -794,104 +770,10 @@ def run_s3_export( if dry_run: logger.info("DRY RUN: Skipping actual upload") - # Save dry run results to YAML file - dry_run_data = { - "s3_export_dry_run": { - "timestamp": datetime.now().isoformat(), - "export_config": { - "bucket": bucket, - "instance": instance, - "directory": directory, - "upload_id": upload_id, - "exported_path": f"s3://{bucket}/{export_s3_prefix}", - }, - "csv_files": [], - "kpis_json_files": [], - "ai_data_files": [], - "upload_plan": upload_plan, - "summary": { - "total_files": len(upload_files), - "total_size_bytes": total_size, - "csv_files_count": len([f for f in upload_files if f.suffix == ".csv"]), - "kpis_json_files_count": len( - [f for f in upload_files if f.name == "kpis.json"] - ), - "ai_data_files_count": len( - [f for f in upload_files if ai_data_dir and ai_data_dir in f.parents] - ), - }, - } - } - - # Collect CSV file details (use already collected files) - for csv_file in csv_files: - try: - relative_csv_path = str(csv_file.relative_to(artifact_dir)) - except ValueError: - relative_csv_path = str(csv_file) - - dry_run_data["s3_export_dry_run"]["csv_files"].append( - { - "name": csv_file.name, - "path": relative_csv_path, - "size_bytes": csv_file.stat().st_size, - } - ) - - # Collect KPI JSON file details - if s3_config.include_kpis_json: - kpi_json_files = list_kpi_json_files(output_dir, postprocess_config) - for kpi_file in kpi_json_files: - try: - relative_kpi_path = str(kpi_file.relative_to(artifact_dir)) - except ValueError: - relative_kpi_path = str(kpi_file) - - dry_run_data["s3_export_dry_run"]["kpis_json_files"].append( - { - "name": kpi_file.name, - "path": relative_kpi_path, - "size_bytes": kpi_file.stat().st_size, - } - ) - - # Collect AI eval file details - if s3_config.include_ai_data and ai_data_dir: - ai_data_files = list_ai_data_files(ai_data_dir) - for ai_data_file in ai_data_files: - relative_path = ai_data_file.relative_to(ai_data_dir) - try: - relative_full_path = str(ai_data_file.relative_to(artifact_dir)) - except ValueError: - relative_full_path = str(ai_data_file) - - dry_run_data["s3_export_dry_run"]["ai_data_files"].append( - { - "full_path": relative_full_path, - "size_bytes": ai_data_file.stat().st_size, - } - ) - - # Write to YAML file - dry_run_file = output_dir / "upload_dry_run.yaml" - try: - with open(dry_run_file, "w") as f: - yaml.dump(dry_run_data, f, default_flow_style=False, indent=2) - logger.info(f"Dry run results saved to: {dry_run_file}") - except Exception as e: - logger.error(f"Failed to save dry run results: {e}") - return { - "status": "failed", - "error": f"Failed to save dry run results: {e}", - "dry_run": True, - "completed_at": time.time(), - "duration": round(time.time() - start_time), - } - + status = "failed" if error_detected else "success" response = { - "status": "success", + "status": status, "dry_run": True, - "dry_run_file": str(dry_run_file), "completed_at": time.time(), "duration": round(time.time() - start_time), } @@ -938,10 +820,11 @@ def run_s3_export( "error": f"All uploads failed. Failed files: {failed_uploads}", "completed_at": time.time(), } - else: - logger.warning( - f"Partial success: {uploaded_count} uploaded, {len(failed_uploads)} failed" - ) + + error_detected = True + logger.warning( + f"Partial success: {uploaded_count} uploaded, {len(failed_uploads)} failed" + ) target_url = f"s3://{bucket}/{export_s3_prefix}" logger.info( @@ -949,10 +832,8 @@ def run_s3_export( ) # Determine final status - if failed_uploads: - status = "partial_success" if uploaded_count > 0 else "failed" - else: - status = "success" + + status = "failed" if error_detected else "success" return { "status": status, From da0f3b3fa64a7e172af39f5c013d44b491da0d37 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 11:36:49 +0200 Subject: [PATCH 0452/1072] [guidellm] postprocess: guidellm: parsing/kpis: remove irrelevant KPI --- .../guidellm/postprocess/guidellm/parsing/kpis.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index 2c0172c88..439cb8943 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -19,17 +19,6 @@ from projects.caliper.engine.model import UnifiedRunModel -@HigherBetter() -@Format("{:.1f}") -@KPIMetadata(help="Request concurrency level", unit="connections") -def guidellm_request_concurrency(unified_record) -> float: - """Request Concurrency KPI.""" - value = unified_record.metrics.get("request_concurrency") - if value is None: - raise ValueError("request_concurrency metric not found") - return float(value) - - # Token Count Statistics KPIs - static values @LowerBetter() @Format("{:.1f}") From 48db99bb2922f92ccce639c4d47d8a58c1fe28e1 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 12:33:54 +0200 Subject: [PATCH 0453/1072] [guidellm] postprocess: Address code-rabbit review --- .../postprocess/guidellm/parsing/parsers.py | 6 +- .../guidellm/tests/test_postprocess_parser.py | 66 +++++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index bee45920b..62991fae7 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -343,7 +343,7 @@ def get_metric_value( cooldown_time=0.0, # Not available in JSON format # Request metrics request_rate=request_rate, - request_concurrency=intended_concurrency, # For now, effective = intended + request_concurrency=intended_concurrency, intended_concurrency=int(intended_concurrency), completed_requests=completed_requests, failed_requests=0, # Could extract from unsuccessful metrics if needed @@ -452,8 +452,8 @@ def _create_aggregated_metrics( if not benchmark_group: return {} - # Sort benchmarks by request rate for consistent curve ordering - sorted_benchmarks = sorted(benchmark_group, key=lambda b: b.request_rate) + # Sort benchmarks by intended concurrency for consistent curve ordering + sorted_benchmarks = sorted(benchmark_group, key=lambda b: b.intended_concurrency) # Use the first benchmark for static characteristics representative = sorted_benchmarks[0] diff --git a/projects/guidellm/tests/test_postprocess_parser.py b/projects/guidellm/tests/test_postprocess_parser.py index 4d66a0193..b26999925 100644 --- a/projects/guidellm/tests/test_postprocess_parser.py +++ b/projects/guidellm/tests/test_postprocess_parser.py @@ -7,7 +7,14 @@ from projects.guidellm.postprocess.guidellm.parsing.parsers import GuideLLMParser -def _write_benchmark_file(path: Path, streams: int) -> None: +def _write_benchmark_file(path: Path, streams: int, request_rate: float | None = None) -> None: + # Use provided request_rate or default to 1.5 * streams for variation + effective_rate = request_rate if request_rate is not None else 1.5 * streams + + # Create different token rates based on streams for ordering verification + input_tokens_rate = 2.0 + streams # e.g., 10, 18, 34 for streams 8, 16, 32 + output_tokens_rate = 1.0 + streams # e.g., 9, 17, 33 for streams 8, 16, 32 + payload = { "args": {"rate": streams}, "metadata": {"label": f"rate-{streams}"}, @@ -16,9 +23,9 @@ def _write_benchmark_file(path: Path, streams: int) -> None: "config": {"strategy": {"type_": "concurrent", "streams": streams}}, "scheduler": {"state": {"start_time": 0, "end_time": 10}}, "metrics": { - "requests_per_second": {"successful": {"mean": 1.5}}, - "input_tokens_per_second": {"successful": {"mean": 3.0}}, - "output_tokens_per_second": {"successful": {"mean": 4.0}}, + "requests_per_second": {"successful": {"mean": effective_rate}}, + "input_tokens_per_second": {"successful": {"mean": input_tokens_rate}}, + "output_tokens_per_second": {"successful": {"mean": output_tokens_rate}}, "request_latency": { "successful": {"median": 100.0, "percentiles": {"p95": 120.0}} }, @@ -83,3 +90,54 @@ def test_parser_accepts_rate_split_benchmark_files(tmp_path: Path) -> None: curves = record.metrics["performance_curves"] assert curves["intended_concurrency"] == [32, 64] assert record.run_identity == {"guidellm": True} + + +def test_parser_orders_by_intended_concurrency_not_request_rate(tmp_path: Path) -> None: + """Test that benchmarks are ordered by intended_concurrency even when request_rate order differs.""" + test_dir = tmp_path / "run" + test_dir.mkdir() + + # Create benchmarks where request_rate order differs from intended_concurrency order + # intended_concurrency: 8, 16, 32 (ascending) + # request_rate: 100, 50, 75 (not ascending) + file_a = test_dir / "benchmarks-rate-8.json" + file_b = test_dir / "benchmarks-rate-16.json" + file_c = test_dir / "benchmarks-rate-32.json" + + _write_benchmark_file(file_a, streams=8, request_rate=100.0) # highest rate, lowest concurrency + _write_benchmark_file(file_b, streams=16, request_rate=50.0) # lowest rate, middle concurrency + _write_benchmark_file(file_c, streams=32, request_rate=75.0) # middle rate, highest concurrency + + parser = GuideLLMParser() + node = TestBaseNode( + directory=test_dir, + test_labels={"labels": {"guidellm_loadshape": "multi-turn"}}, + artifact_paths=[file_a, file_b, file_c], + ) + + result = parser.parse([node]) + + assert result.warnings == [] + assert len(result.records) == 1 + + # Check that curves are ordered by intended_concurrency (8, 16, 32) not request_rate (50, 75, 100) + record = result.records[0] + curves = record.metrics["performance_curves"] + + # Verify intended_concurrency curve follows intended order + assert curves["intended_concurrency"] == [8, 16, 32] + + # Verify tokens_per_second curve follows intended_concurrency order (not rate order) + # For streams 8: input_tokens=10, output_tokens=9 → total=19 + # For streams 16: input_tokens=18, output_tokens=17 → total=35 + # For streams 32: input_tokens=34, output_tokens=33 → total=67 + # Should be ordered by streams (8, 16, 32), not by request_rate (50, 75, 100) + assert curves["tokens_per_second"] == [19.0, 35.0, 67.0] + + # Verify request rates are in intended_concurrency order, not ascending rate order + request_rates = record.metrics["request_rate"] + assert request_rates == [ + 100.0, + 50.0, + 75.0, + ] # Rates in intended_concurrency order (8→100, 16→50, 32→75) From d95b8604b71a9ccda9c87c42e2c4292241e50544 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 6 Aug 2026 13:20:31 -0400 Subject: [PATCH 0454/1072] fix: walk up directory tree to find test labels for mlflow_destination The KPI run_path points deep inside the benchmark results, several levels below where __test_labels__.yaml is written. Walk up from run_path instead of only checking the immediate parent. Co-authored-by: Cursor --- projects/rhaiis/postprocess/plugin.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index b396c0e36..a52c3d521 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -20,25 +20,26 @@ def _read_mlflow_destination_from_test_labels(kpi_records: list[dict]) -> dict[str, str]: - """Extract mlflow_destination from __test_labels__.yaml near the first KPI run_path.""" + """Extract mlflow_destination from __test_labels__.yaml by walking up from KPI run_path.""" import yaml for kpi in kpi_records: run_path = kpi.get("run_path", "") if not run_path: continue - labels_file = Path(run_path) / "__test_labels__.yaml" - if not labels_file.exists(): - labels_file = Path(run_path).parent / "__test_labels__.yaml" - if not labels_file.exists(): - continue - try: - data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) - dest = data.get("mlflow_destination") if isinstance(data, dict) else None - if isinstance(dest, dict) and dest.get("run_id"): - return dest - except (OSError, yaml.YAMLError): - pass + current = Path(run_path) + while current != current.parent: + labels_file = current / "__test_labels__.yaml" + if labels_file.exists(): + try: + data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) + dest = data.get("mlflow_destination") if isinstance(data, dict) else None + if isinstance(dest, dict) and dest.get("run_id"): + return dest + except (OSError, yaml.YAMLError): + pass + break + current = current.parent return {} From e8df4379a02dcfcbc916ddda1ab3412aeb2d6cfe Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 6 Aug 2026 13:50:32 -0400 Subject: [PATCH 0455/1072] fix: pass mlflow IDs through KPI labels instead of filesystem reads run_path in KPI records is relative, so the CSV plugin subprocess cannot resolve __test_labels__.yaml on disk. Instead, extract mlflow_destination from test nodes during compute_kpis and include mlflow_run_id/mlflow_experiment_id in KPI record labels. Co-authored-by: Cursor --- projects/rhaiis/postprocess/kpis.py | 9 ++++++++ projects/rhaiis/postprocess/plugin.py | 30 ++------------------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index fc137feea..1e8c7ece2 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -63,6 +63,13 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") out: list[dict[str, Any]] = [] + mlflow_dest = {} + for node in model.test_nodes: + dest = node.test_labels.get("mlflow_destination") + if isinstance(dest, dict) and dest.get("run_id"): + mlflow_dest = dest + break + for r in model.unified_result_records: if not r.run_identity.get("guidellm"): continue @@ -80,6 +87,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: "output_toks": str(r.metrics.get("output_toks", "")), "guidellm_start_time_ms": str(r.metrics.get("guidellm_start_time_ms", "")), "guidellm_end_time_ms": str(r.metrics.get("guidellm_end_time_ms", "")), + "mlflow_run_id": mlflow_dest.get("run_id", ""), + "mlflow_experiment_id": mlflow_dest.get("experiment_id", ""), } if not curves or not request_rates: diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index a52c3d521..0ae902860 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -19,30 +19,6 @@ logger = logging.getLogger(__name__) -def _read_mlflow_destination_from_test_labels(kpi_records: list[dict]) -> dict[str, str]: - """Extract mlflow_destination from __test_labels__.yaml by walking up from KPI run_path.""" - import yaml - - for kpi in kpi_records: - run_path = kpi.get("run_path", "") - if not run_path: - continue - current = Path(run_path) - while current != current.parent: - labels_file = current / "__test_labels__.yaml" - if labels_file.exists(): - try: - data = yaml.safe_load(labels_file.read_text(encoding="utf-8")) - dest = data.get("mlflow_destination") if isinstance(data, dict) else None - if isinstance(dest, dict) and dest.get("run_id"): - return dest - except (OSError, yaml.YAMLError): - pass - break - current = current.parent - return {} - - # CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. # GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. _SECONDS_TO_MS_COLUMNS = frozenset( @@ -150,8 +126,6 @@ def export_kpis_to_csv( """ from projects.rhaiis.postprocess.csv_export import FIELDNAMES - mlflow_dest = _read_mlflow_destination_from_test_labels(kpi_records) - # Group KPIs by (run_path, rate_index) groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(dict) group_labels: dict[tuple[str, str], dict[str, Any]] = {} @@ -215,8 +189,8 @@ def export_kpis_to_csv( row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") row["guidellm_version"] = labels.get("guidellm_version", "") - row["mlflow_run_id"] = mlflow_dest.get("run_id", "") - row["mlflow_experiment_id"] = mlflow_dest.get("experiment_id", "") + row["mlflow_run_id"] = labels.get("mlflow_run_id", "") + row["mlflow_experiment_id"] = labels.get("mlflow_experiment_id", "") rows.append(row) output_path.parent.mkdir(parents=True, exist_ok=True) From af6bc4c0b3912a610301d0639f7e145fbf172f20 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Thu, 6 Aug 2026 19:57:19 +0100 Subject: [PATCH 0456/1072] fix: Rename control plane component Signed-off-by: Alberto Perdomo --- projects/kserve/toolbox/wait_kserve_ready/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py index 426cf2355..74e9ac30a 100644 --- a/projects/kserve/toolbox/wait_kserve_ready/main.py +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -8,7 +8,7 @@ from projects.core.dsl.utils.k8s import oc SERVING_CONTROL_PLANE_DEPLOYMENTS = ( - "kserve-controller-manager", + "kserve-module-controller-manager", "llmisvc-controller-manager", "model-serving-api", "odh-model-controller", From d905471a1895099c20430fc923dd7f11f75e88c7 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 6 Aug 2026 15:54:08 -0400 Subject: [PATCH 0457/1072] refactor: move _build_mlflow_run_url wrapper to caliper with @requires The config-aware wrapper only uses caliper config keys, so it belongs in caliper/orchestration/export.py where all projects can reuse it. regression.py now imports and calls build_mlflow_run_url_from_config(). Co-authored-by: Cursor --- projects/caliper/orchestration/export.py | 25 +++++++++++++++++++++ projects/rhaiis/postprocess/regression.py | 27 +++++------------------ 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 7d3866ca3..c9a5b4569 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -400,6 +400,31 @@ def _read_mlflow_ids_from_test_labels() -> tuple[str, str]: return "", "" +@requires( + vault_name="caliper.export.backend.mlflow.secrets.vault.name", + vault_key="caliper.export.backend.mlflow.secrets.vault.mlflow_secret", + workspace="caliper.export.backend.mlflow.config.workspace", +) +def build_mlflow_run_url_from_config(_cfg) -> str: + """Config-aware wrapper around :func:`build_mlflow_run_url`. + + Resolves vault secrets and workspace from project config via ``@requires``. + Returns an empty string if MLflow is not configured or URL cannot be built. + """ + vault_name = _cfg.vault_name + vault_key = _cfg.vault_key + if not vault_name or not vault_key: + logger.warning("Cannot build MLflow URL: vault not configured") + return "" + + secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) + if not secrets_path or not secrets_path.exists(): + logger.warning("Cannot build MLflow URL: secrets file not found") + return "" + + return build_mlflow_run_url(secrets_path=secrets_path, workspace=_cfg.workspace or None) + + def build_mlflow_run_url( *, secrets_path: Path, diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index 6d4096b4f..3f575ae9f 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -334,31 +334,14 @@ def run_regression_analysis( def _build_mlflow_run_url() -> str: """Construct the MLflow run URL at runtime from vault secrets and config.""" - from projects.caliper.orchestration.export import build_mlflow_run_url - from projects.core.library import config - from projects.core.library import vault as vault_lib + from projects.caliper.orchestration.export import build_mlflow_run_url_from_config - vault_name = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.name", None, print=False, warn=False - ) - vault_key = config.project.get_config( - "caliper.export.backend.mlflow.secrets.vault.mlflow_secret", None, print=False, warn=False - ) - if not vault_name or not vault_key: - logger.warning("Cannot build MLflow URL: vault not configured") - return "" - - secrets_path = vault_lib.get_vault_content_path(vault_name, vault_key) - if not secrets_path or not secrets_path.exists(): - logger.warning("Cannot build MLflow URL: secrets file not found") + try: + return build_mlflow_run_url_from_config() + except Exception: + logger.warning("Failed to build MLflow run URL", exc_info=True) return "" - workspace = config.project.get_config( - "caliper.export.backend.mlflow.config.workspace", None, print=False, warn=False - ) - - return build_mlflow_run_url(secrets_path=secrets_path, workspace=workspace) - PROFILE_DISPLAY_NAMES = { "profile1": "Profile A: Balanced (1k/1k)", From 3de0d58dcf518aba50361b30d71dd90470c07b83 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 5 Aug 2026 14:11:24 +0200 Subject: [PATCH 0458/1072] [guidellm] postprocess: guidellm: include a report_deployment_profile report --- .../guidellm/plotting/performance_analysis.py | 605 +++++++++++++++++- .../guidellm/postprocess/guidellm/plugin.py | 19 +- .../guidellm/visualize-groups.yaml | 6 + 3 files changed, 601 insertions(+), 29 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 6e8ef1c78..5409c609a 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -22,6 +22,18 @@ logger = logging.getLogger(__name__) +# Filesystem-unsafe characters for path sanitization +_PATH_UNSAFE_CHARS = ["/", "\\", ":", "*", "?", "|", "<", ">", '"'] + + +def sanitize_for_path(text: str) -> str: + """Replace filesystem-unsafe characters in label values.""" + result = str(text) + for char in _PATH_UNSAFE_CHARS: + result = result.replace(char, "_") + return result + + def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: Any = 0.0) -> Any: """ Safely extract a value from a performance curve at a specific index. @@ -273,7 +285,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if rate_points > 1 else f"{config_data['request_rate'].iloc[0]:.1f} req/s" ) - logger.info(f" • {config}: {rate_points} rate points ({rate_range})") + logger.info(f" - {config}: {rate_points} rate points ({rate_range})") return df @@ -629,6 +641,548 @@ def generate_token_throughput_percentiles_analysis( ) +def generate_deployment_profile_report( + records: list[UnifiedResultRecord], + output_dir: Path, + title_context: str = "", + report_number: int | None = None, + report_title: str = "GuideLLM Deployment Profile Analysis", +) -> str | None: + """ + Generate performance analysis report with separate plots for comparison groups. + + Groups records by identical test conditions (all labels except the version key) and only + creates plots for groups that have records with different version values. + This enables comparing different versions under identical test conditions. + + Args: + records: List of unified result records + output_dir: Directory to save files + title_context: Additional context for titles + report_number: Optional report number for file naming (e.g., 0 for "Report 00:") + report_title: Title for the report (used in filename and HTML title) + """ + try: + # Create report identifier using core utilities + display_title = create_report_title_display(report_title, report_number) + + logger.info(f"\n🚀 Starting {display_title} generation...") + logger.info("=" * 70) + + # Use version as the comparison key to distinguish results + comparison_keys = {"version"} + + # First, analyze what keys are actually available in the data + all_keys = set() + comparison_key_values = {key: set() for key in comparison_keys} + + for record in records: + # Use distinguishing labels directly for analysis + labels = record.distinguishing_labels + + # Debug: Print version values found + if "version" in labels: + logger.debug( + f"🐛 DEBUG: Found version='{labels['version']}' in record {record.test_base_path}" + ) + else: + logger.debug( + f"🐛 DEBUG: No 'version' key found in record {record.test_base_path}, keys: {list(labels.keys())}" + ) + + all_keys.update(labels.keys()) + for key in comparison_keys: + if key in labels: + comparison_key_values[key].add(labels[key]) + + logger.info("\n📊 Data analysis for comparison report:") + logger.info(f" Total records: {len(records)}") + logger.info(f" Available label keys: {sorted(all_keys)}") + logger.info(" Comparison key availability:") + for key in comparison_keys: + values = comparison_key_values[key] + logger.info( + f" {key}: {len(values)} unique values: {sorted(values) if values else 'NOT FOUND'}" + ) + + # If version key has multiple values, use it; otherwise try deployment_profile fallback + active_comparison_keys = [] + for key in comparison_keys: + if len(comparison_key_values[key]) > 1: + active_comparison_keys.append(key) + + if not active_comparison_keys: + logger.info("⚠️ No version key with multiple values found, trying fallback keys...") + fallback_keys = ["deployment_profile", "guidellm_loadshape"] + fallback_values = {} + + for record in records: + # Use distinguishing labels directly for fallback logic + labels = record.distinguishing_labels + + for key in fallback_keys: + if key not in fallback_values: + fallback_values[key] = set() + + # Use distinguishing labels directly + value = labels.get(key, "default") + + fallback_values[key].add(value) + + logger.info(" Fallback key availability:") + for key in fallback_keys: + values = fallback_values[key] + logger.info(f" {key}: {len(values)} unique values: {sorted(values)}") + if len(values) > 1: + active_comparison_keys.append(key) + + if not active_comparison_keys: + logger.info("⚠️ No suitable version values found for comparison") + logger.info(" 📊 Generating single-dataset report...") + + # Use a placeholder comparison key set for single-group analysis + active_comparison_keys = ["dataset"] + comparison_keys = {"dataset"} + + logger.info(f" Using comparison keys: {active_comparison_keys}") + comparison_keys = set(active_comparison_keys) + + # Create one unified group for version comparison across all test configurations + core_labels = all_keys + + comparison_groups = {} + + for record in records: + # Create a group key from only core distinguishing labels (not version or config details) + group_labels = {} + for k, v in record.distinguishing_labels.items(): + if k not in comparison_keys and k in core_labels: + group_labels[k] = v + + # Convert to a sortable tuple for grouping - if no core labels, use a single group + if group_labels: + group_key = tuple(sorted(group_labels.items())) + else: + group_key = ("unified_comparison",) + + if group_key not in comparison_groups: + comparison_groups[group_key] = [] + comparison_groups[group_key].append(record) + + # Filter to only groups that have records with different comparison key values + valid_groups = {} + for group_key, group_records in comparison_groups.items(): + # Extract comparison key values for this group + comparison_values = set() + for record in group_records: + # Use distinguishing labels directly for comparison key evaluation + comp_values = [] + for key in comparison_keys: + value = record.distinguishing_labels.get(key, "unknown") + comp_values.append(value) + comparison_values.add(tuple(comp_values)) + + # Only include groups with multiple comparison key values + if len(comparison_values) > 1: + valid_groups[group_key] = group_records + + if not valid_groups: + logger.info( + f"⚠️ No comparison groups found - all records have identical values for keys: {active_comparison_keys}" + ) + logger.info(" 📊 Generating single-group report with all data...") + + # Create a single group with all records and include warning in the description + warning_message = f"No comparisons possible - all records have identical {', '.join(active_comparison_keys)} values" + single_group_key = ("all_data", warning_message) + valid_groups = {single_group_key: records} + + logger.info( + f"\n🔍 Found {len(valid_groups)} comparison groups with varying {active_comparison_keys}:" + ) + for i, (group_key, group_records) in enumerate(valid_groups.items(), 1): + # Handle special case for warning message + if len(group_key) == 2 and group_key[0] == "all_data": + group_desc = "All Available Data" + else: + group_desc = ", ".join(f"{k}={v}" for k, v in group_key) + comp_values = set() + for record in group_records: + # Use distinguishing labels directly for comparison key evaluation + comp_vals = [] + for key in comparison_keys: + value = record.distinguishing_labels.get(key, "unknown") + comp_vals.append(value) + comp_values.add(tuple(comp_vals)) + comp_desc = " vs ".join(f"{':'.join(cv)}" for cv in sorted(comp_values)) + logger.info(f" 📋 Group {i}: [{group_desc}] comparing [{comp_desc}]") + + # Generate all the plots as figures + plot_functions = [ + ("Token Throughput vs Concurrency", create_token_throughput_vs_concurrency_plot), + ("TTFT Analysis", create_ttft_analysis_plot), + ("Token Throughput Percentiles", create_token_throughput_percentiles_plot), + ("Throughput Scaling", create_throughput_scaling_plot), + ("Latency vs Throughput", create_latency_vs_throughput_plot), + ] + + # Create dedicated directory for this report + if report_number is not None: + report_dir_name = f"report_{report_number:02d}_comparison_analysis" + else: + report_dir_name = "comparison_analysis" + + report_dir = output_dir / report_dir_name + report_dir.mkdir(exist_ok=True) + logger.info(f"\n📁 Created report directory: {report_dir_name}") + + all_plots_data = [] + + # Process each comparison group separately + for group_key, group_records in valid_groups.items(): + # Create group description for display and directory naming + # Handle special case for warning message + if len(group_key) == 2 and group_key[0] == "all_data": + group_desc = "All Available Data" + group_name = "all_data" + warning_msg = group_key[1] # Extract warning message + else: + group_desc = ", ".join(f"{k}={v}" for k, v in group_key) + group_name = "__".join(f"{k}_{sanitize_for_path(v)}" for k, v in group_key) + warning_msg = None + + logger.info(f"\n📊 Processing comparison group: {group_desc}") + logger.info(f" Records: {len(group_records)}") + + # Create DataFrame for this comparison group + df = create_dataframe_from_records(group_records) + if df.empty: + logger.info(f" ⚠️ No data available for group: {group_desc}") + continue + + logger.info(f" 📈 Generating {len(plot_functions)} plots for {group_desc}...") + + # Create subdirectory for this comparison group + group_dir = report_dir / group_name + group_dir.mkdir(exist_ok=True) + + # Extract metadata for this group + comp_values = set() + for r in group_records: + comp_tuple = tuple( + r.distinguishing_labels.get(key, "unknown") for key in comparison_keys + ) + comp_values.add(comp_tuple) + comp_desc = " vs ".join(f"{':'.join(cv)}" for cv in sorted(comp_values)) + + group_plots = [] + for i, (plot_name, plot_func) in enumerate(plot_functions, 1): + logger.info( + f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for comparison group..." + ) + try: + # Create subtitle with group description and comparison info + if warning_msg: + # Show warning message instead of comparison info + subtitle = f"{group_desc} - ⚠️ {warning_msg}" + else: + subtitle = f"Comparison Group: {group_desc} | Comparing: {comp_desc}" + + # Add original title context if provided + if title_context: + group_title_context = f"{title_context}
{subtitle}" + else: + group_title_context = f"
{subtitle}" + + fig = plot_func(df, group_title_context) + if fig: + # Save as both PNG and HTML in the group directory + filename = f"{group_name}_{plot_name.lower().replace(' ', '_')}" + + # Save PNG image + width = 900 if "Percentiles" in plot_name else 800 + height = 600 if "Percentiles" in plot_name else 500 + png_path = save_figure( + fig, group_dir, filename, as_image=True, width=width, height=height + ) + + # Save HTML version + html_path = save_figure(fig, group_dir, filename, as_image=False) + + if png_path and html_path: + # Store relative paths for linking + group_plots.append( + ( + plot_name, + f"{report_dir_name}/{group_name}/{Path(png_path).name}", # PNG path + f"{report_dir_name}/{group_name}/{Path(html_path).name}", # HTML path + ) + ) + logger.info(f" ✅ {plot_name} saved for comparison group") + else: + logger.info( + f" ⚠️ {plot_name} could not be created for comparison group (no figure returned)" + ) + + except Exception as e: + logger.info(f" ❌ Failed to generate {plot_name} for comparison group: {e}") + + # Store group plots data if any plots were created + if group_plots: + # Include group_key for displaying identical labels + group_labels_dict = dict(group_key) if group_key != ("unified_comparison",) else {} + all_plots_data.append((group_desc, group_plots, group_labels_dict)) + + if not all_plots_data: + logger.info("❌ No plots were successfully generated") + return None + + logger.info( + f"\n✅ Successfully generated plots for {len(all_plots_data)} comparison group(s)!" + ) + + # Create comprehensive HTML report with sections for each comparison group + logger.info("\n📝 Assembling comprehensive HTML report...") + logger.info( + " 🔗 Creating report with comparison group sections and interactive HTML links..." + ) + + html_content = f""" + + + + + + {display_title} + + + + + +

Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.

+ +
+

{display_title}

+

Generated performance analysis with separate visualizations for each comparison group

+
+ +
+

Quick Navigation:

+
    """ + + # Add navigation links for each comparison group + for group_desc, _, _ in all_plots_data: + group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") + html_content += f'\n
  • {group_desc}
    • ' + + html_content += """ +
+
+""" + # Add sections for each comparison group + for group_desc, plots, group_labels in all_plots_data: + group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") + + # Handle special warning case + if group_desc == "All Available Data": + title_prefix = "Dataset:" + explanation = ( + "⚠️ No comparisons possible - all records have identical version values" + ) + else: + title_prefix = "Comparison Group:" + explanation = f"Comparing different values of {active_comparison_keys} across identical test conditions" + + # Generate display text for all identical labels + if group_labels: + labels_html = ( + "
    " + + "".join( + f"
  • {k}: {v}
    • " + for k, v in sorted(group_labels.items()) + ) + + "
" + ) + else: + labels_html = "

All data (no grouping constraints)

" + + html_content += f""" +
+

{title_prefix} {group_desc}

+

+ {explanation} +

+ +
+ Identical labels: + {labels_html} +
+ + """ + + # Find only the throughput plot + throughput_plot = None + for plot_name, png_path, html_path in plots: + if plot_name == "Token Throughput vs Concurrency": + throughput_plot = (plot_name, png_path, html_path) + break + + if throughput_plot: + plot_name, png_path, html_path = throughput_plot + html_content += f""" +
+

🚀 {plot_name}

+

Token generation throughput scaling analysis across different concurrency levels.

+

{plot_name}

+
+
""" + else: + html_content += """ +
+

⚠️ No throughput plot available for this group.

+
+ """ + + # Close the HTML after all groups are processed + html_content += """ + +""" + + # Save the main HTML report + main_html_filename = create_report_filename( + "comparison_analysis", report_number, report_title, "html" + ) + main_html_path = output_dir / main_html_filename + + with open(main_html_path, "w", encoding="utf-8") as f: + f.write(html_content) + + logger.info(f"✅ Comparison analysis report saved as: {main_html_filename}") + logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") + + logger.info("=" * 70) + logger.info(f"🎉 {display_title} ready: {main_html_path.name}") + + return str(main_html_path) + + except Exception: + logger.exception("❌ Failed to generate deployment profile report") + # Re-raise for debugging - this error should not be silently ignored + raise + + def generate_comprehensive_performance_report( records: list[UnifiedResultRecord], output_dir: Path, @@ -704,7 +1258,7 @@ def generate_comprehensive_performance_report( logger.info(f" 📈 Generating {len(plot_functions)} plots for {loadshape}...") # Create subdirectory for this loadshape - loadshape_dir = report_dir / loadshape + loadshape_dir = report_dir / sanitize_for_path(loadshape) loadshape_dir.mkdir(exist_ok=True) # Extract model information for this loadshape @@ -727,7 +1281,7 @@ def generate_comprehensive_performance_report( ) try: # Create subtitle with model and loadshape info - subtitle = f"Model: {model_info} • Load Shape: {loadshape}" + subtitle = f"Model: {model_info} | Load Shape: {loadshape}" # Add original title context if provided if title_context: @@ -790,15 +1344,15 @@ def generate_comprehensive_performance_report( {display_title} + + + + {% block extra_head %}{% endblock %} + + + +
+
+

{% block header_title %}Caliper Report{% endblock %}

+

{% block header_subtitle %}Performance Analysis Report{% endblock %}

+
+ + {% block content %}{% endblock %} +
+ + {% block extra_js %}{% endblock %} + + \ No newline at end of file diff --git a/projects/caliper/templates/kpi_dashboard.html b/projects/caliper/templates/kpi_dashboard.html new file mode 100644 index 000000000..dcc8e7b95 --- /dev/null +++ b/projects/caliper/templates/kpi_dashboard.html @@ -0,0 +1,198 @@ +{% extends "base.html" %} + +{% block title %}Caliper KPI Dashboard{% endblock %} + +{% block header_title %}KPI Dashboard{% endblock %} + +{% block header_subtitle %}Performance Metrics Report - {{ total_tests }} test(s), {{ total_kpis }} KPI(s){% endblock %} + +{% block content %} + +
+
+

{{ total_tests }}

+

Test Runs

+
+
+

{{ total_kpis }}

+

Total KPIs

+
+
+

{{ curve_kpis_count }}

+

Curve KPIs

+
+
+

{{ scalar_kpis_count }}

+

Scalar KPIs

+
+
+ + +{% for test in tests %} +
+

Test Run: {{ test.run_id }}

+ + {% if test.metadata and test.metadata.timestamp %} +

Generated: {{ test.metadata.timestamp }}

+ {% endif %} + + + {% if test.labels %} +
+ Labels: +
+ {% for key, value in test.labels.items() %} + {{ key }}: {{ value }} + {% endfor %} +
+
+ {% endif %} + + + + + + + + + + + + + + + {% for kpi in test.kpis %} + + + + + + + + + {% endfor %} + +
KPIValueUnitTypeTrendDescription
{{ kpi.name or kpi.kpi_id }} + {% if kpi.is_curve %} + {% if kpi.values_count > 0 %} + {{ kpi.values_count }} points + {% else %} + No data + {% endif %} + {% else %} + {% if kpi.value is not none %} + {{ "%.3f"|format(kpi.value) if kpi.value is number else kpi.value }} + {% else %} + No data + {% endif %} + {% endif %} + {{ kpi.unit or '-' }} + {% if kpi.is_curve %} + Curve + {% else %} + Scalar + {% endif %} + + {% if kpi.higher_is_better %} + ↑ Higher Better + {% else %} + ↓ Lower Better + {% endif %} + {{ kpi.help or '-' }}
+ + + {% set curve_kpis = test.kpis | selectattr('is_curve', 'equalto', true) | list %} + {% if curve_kpis %} +

Curve Visualizations

+ {% for kpi in curve_kpis %} + {% if kpi.values %} +
+
+
+ {% endif %} + {% endfor %} + {% endif %} +
+{% endfor %} + +{% if not tests %} +
+

No Data Available

+

No KPI data was found in the provided file.

+
+{% endif %} +{% endblock %} + +{% block extra_js %} + +{% endblock %} \ No newline at end of file diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html new file mode 100644 index 000000000..9132401ee --- /dev/null +++ b/projects/caliper/templates/regression_report.html @@ -0,0 +1,299 @@ +{% extends "base.html" %} + +{% block title %}Caliper Regression Analysis{% endblock %} + +{% block header_title %}Regression Analysis Report{% endblock %} + +{% block header_subtitle %} +Status: {{ overall.verdict }} | {{ tested.regression }} regressions found in {{ tested.total_kpis }} KPIs +{% endblock %} + +{% block content %} + +
+
+

+ {{ overall.verdict.replace('_', ' ').title() }} +

+

Overall Status

+
+
+

{{ tested.total_kpis }}

+

Total KPIs

+
+
+

{{ tested.pass_count }}

+

Passed

+
+
+

{{ tested.regression }}

+

Regressions

+
+ {% if tested.skipped > 0 %} +
+

{{ tested.skipped }}

+

Skipped

+
+ {% endif %} +
+ + +
+

Analysis Information

+ + {% if analysis.timestamp %} +

Analysis completed: {{ analysis.timestamp }}

+ {% endif %} + + + + + + + {% if config.comparison_labels %} + + + + + {% endif %} + {% if config.ignored_labels %} + + + + + {% endif %} + {% if config.sorting_labels %} + + + + + {% endif %} +
ConfigurationValue
Comparison Labels{{ config.comparison_labels | join(', ') }}
Ignored Labels{{ config.ignored_labels | join(', ') }}
Sorting Labels{{ config.sorting_labels | join(', ') }}
+
+ + +{% set regression_results = results | selectattr('verdict', 'equalto', 'REGRESSION') | list %} +{% if regression_results %} +
+

Regressions Detected ({{ regression_results|length }})

+ + + + + + + + + + + + + + {% for result in regression_results %} + + + + + + + + + {% endfor %} + +
KPICurrent ValueBaseline MeanChangeAlgorithmDetails
+ {{ result.kpi_id }} + {% if result.run_id %} +
{{ result.run_id }} + {% endif %} + 		+ {% if result.current_value %} + {% if result.is_curve %} + {% if result.current_value.values %} + {{ result.current_value.values|length }} points + {% else %} + No data + {% endif %} + {% else %} + {% if result.current_value.value is not none %} + {{ "%.3f"|format(result.current_value.value) if result.current_value.value is number else result.current_value.value }} + {% else %} + No data + {% endif %} + {% endif %} + {% else %} + No data + {% endif %} + + {% if result.details and result.details.baseline_mean is defined %} + {{ "%.3f"|format(result.details.baseline_mean) if result.details.baseline_mean is number else result.details.baseline_mean }} + {% else %} + - + {% endif %} + + {% if result.details and result.details.relative_change is defined %} + + {{ "%+.1f%%"|format(result.details.relative_change * 100) }} + + {% else %} + - + {% endif %} + + {{ result.details.algorithm if result.details and result.details.algorithm else 'Unknown' }} + + {% if result.details %} + + Baselines: {{ result.baseline_count }}
+ {% if result.details.config %} + Threshold: {{ result.details.config.max_relative_regression if result.details.config.max_relative_regression is defined else 'N/A' }} + {% endif %} + + {% else %} + - + {% endif %} +
+
+{% endif %} + + +
+

All Test Results

+ + + + + + + + + + + + + {% for result in results %} + + + + + + + + {% endfor %} + +
KPIVerdictCurrent ValueBaselinesLabels
+ {{ result.kpi_id }} + {% if result.run_id %} +
{{ result.run_id }} + {% endif %} + 		+ + {{ result.verdict }} + + + {% if result.current_value %} + {% if result.is_curve %} + {% if result.current_value.values %} + {{ result.current_value.values|length }} points + {% else %} + No data + {% endif %} + {% else %} + {% if result.current_value.value is not none %} + {{ "%.3f"|format(result.current_value.value) if result.current_value.value is number else result.current_value.value }} + {% else %} + No data + {% endif %} + {% endif %} + {% else %} + No data + {% endif %} + {{ result.baseline_count }} + {% if result.labels and result.labels.comparison_keys %} +
+ {% for key, value in result.labels.comparison_keys.items() %} + {{ key }}: {{ value }} + {% endfor %} +
+ {% else %} + - + {% endif %} +
+
+ + +{% set regression_data = results | selectattr('verdict', 'equalto', 'REGRESSION') | selectattr('details.relative_change', 'defined') | list %} +{% if regression_data %} +
+

Regression Trend Visualization

+
+
+
+
+{% endif %} + +{% if not results %} +
+

No Results Available

+

No regression analysis results were found in the provided file.

+
+{% endif %} +{% endblock %} + +{% block extra_js %} +{% if regression_data %} + +{% endif %} +{% endblock %} \ No newline at end of file From 1915a255046180f338b93876d0ceea52abff1171 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 09:05:00 +0200 Subject: [PATCH 0758/1072] AGENTS: lint --- AGENTS.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a2ea27225..4dea0e382 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,10 @@ def safe_oc_apply(artifact_path, manifest): # GOOD: Pass secret data by file path; the value must never appear in command text run_command( - "create", "secret", "generic", "example-secret", + "create", + "secret", + "generic", + "example-secret", "--from-file=SECRET_VALUE=/secure/temp/path", ) ``` @@ -101,7 +104,9 @@ logger.info(f"Applying secret: {manifest}") # LEAKS if manifest has sensitive d raise RuntimeError(f"Failed to auth with token {token}") # LEAKS TOKEN # BAD: Passing secret material as a command-line literal; command text may be logged -run_command("create", "secret", "generic", "example-secret", "--from-literal=SECRET_VALUE=" + secret) +run_command( + "create", "secret", "generic", "example-secret", "--from-literal=SECRET_VALUE=" + secret +) ``` ## Error Handling: Never Swallow Exceptions From 61f1492d745993b8420b4248b68136e0c57cb7cd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 09:51:13 +0200 Subject: [PATCH 0759/1072] [caliper] validate the KPI plotting --- projects/caliper/engine/kpi/html_generator.py | 12 +- projects/caliper/templates/kpi_dashboard.html | 202 ++++++++++++++---- 2 files changed, 168 insertions(+), 46 deletions(-) diff --git a/projects/caliper/engine/kpi/html_generator.py b/projects/caliper/engine/kpi/html_generator.py index f707e6e2c..dbc98924b 100644 --- a/projects/caliper/engine/kpi/html_generator.py +++ b/projects/caliper/engine/kpi/html_generator.py @@ -52,6 +52,7 @@ def generate_kpi_html(self, kpi_data: dict | list, output_file: Path) -> None: total_kpis = 0 curve_kpis_count = 0 scalar_kpis_count = 0 + curve_kpi_ids = set() # Track unique curve KPI IDs for test in tests: for kpi in test.get("kpis", []): @@ -61,9 +62,13 @@ def generate_kpi_html(self, kpi_data: dict | list, output_file: Path) -> None: # Add values count and prepare chart data values = kpi.get("values", []) kpi["values_count"] = len(values) if values else 0 - # Pre-serialize the values as JSON string for safe template usage + # Store clean values list for template iteration + kpi["values_list"] = list(values) if values else [] + # Pre-serialize the values as JSON string for safe chart usage if values: kpi["values_json"] = json.dumps(values) + # Add to curve KPI IDs set if it has data + curve_kpi_ids.add(kpi.get("kpi_id")) else: kpi["values_json"] = "[]" else: @@ -76,6 +81,7 @@ def generate_kpi_html(self, kpi_data: dict | list, output_file: Path) -> None: total_kpis=total_kpis, curve_kpis_count=curve_kpis_count, scalar_kpis_count=scalar_kpis_count, + curve_kpi_ids=sorted(curve_kpi_ids), # Pass sorted list of curve KPI IDs ) output_file.parent.mkdir(parents=True, exist_ok=True) @@ -174,7 +180,9 @@ def _group_flat_kpis(self, flat_kpis: list[dict]) -> list[dict]: if kpi_data["is_curve"]: values = kpi_data.get("values", []) kpi_data["values_count"] = len(values) if values else 0 - # Pre-serialize the values as JSON string for safe template usage + # Store clean values list for template iteration + kpi_data["values_list"] = list(values) if values else [] + # Pre-serialize the values as JSON string for safe chart usage if values: kpi_data["values_json"] = json.dumps(values) else: diff --git a/projects/caliper/templates/kpi_dashboard.html b/projects/caliper/templates/kpi_dashboard.html index dcc8e7b95..a0cb123f6 100644 --- a/projects/caliper/templates/kpi_dashboard.html +++ b/projects/caliper/templates/kpi_dashboard.html @@ -27,9 +27,46 @@

{{ scalar_kpis_count }}

+ +{% if tests|length > 1 %} +
+

Test Runs

+ + + + + + + + + {% for test in tests %} + + + + + {% endfor %} + +
Test RunLabels
+ + {{ test.run_id }} + + + {% if test.labels %} +
+ {% for key, value in test.labels.items() %} + {{ key }}: {{ value }} + {% endfor %} +
+ {% else %} + - + {% endif %} +
+
+{% endif %} + {% for test in tests %} -
+

Test Run: {{ test.run_id }}

{% if test.metadata and test.metadata.timestamp %} @@ -100,14 +137,31 @@

Test Run: {{ test.run_id }}

- + {% set curve_kpis = test.kpis | selectattr('is_curve', 'equalto', true) | list %} {% if curve_kpis %} -

Curve Visualizations

{% for kpi in curve_kpis %} - {% if kpi.values %} -
-
+ {% if kpi.values_list and kpi.values_list|length > 0 %} +
+
{{ kpi.name or kpi.kpi_id }}
+ + + + + {% for point in kpi.values_list %} + + {% endfor %} + + + + + + {% for point in kpi.values_list %} + + {% endfor %} + + +
{{ kpi.x_help or kpi.x_unit or 'X' }}{{ point[0] }}
{{ kpi.y_help or kpi.unit or 'Y' }}{{ "%.3f"|format(point[1]) if point[1] is number else point[1] }}
{% endif %} {% endfor %} @@ -115,6 +169,32 @@

Curve Visualizations

{% endfor %} + + +{% if curve_kpi_ids %} +
+

Curve Visualizations

+ {% for kpi_id in curve_kpi_ids %} + {% set first_kpi = none %} + {% for test in tests %} + {% for kpi in test.kpis %} + {% if kpi.kpi_id == kpi_id and kpi.is_curve %} + {% if not first_kpi %} + {% set first_kpi = kpi %} + {% endif %} + {% endif %} + {% endfor %} + {% endfor %} +
+

{{ first_kpi.name or kpi_id if first_kpi else kpi_id }}

+
+
+
+
+ {% endfor %} +
+{% endif %} + {% if not tests %}

No Data Available

@@ -125,57 +205,89 @@

No Data Available

{% block extra_js %} -{% endblock %} \ No newline at end of file +{% endblock %} From 705a1a75880abc7f163825ca8afcafd19060d5a5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 11:08:31 +0200 Subject: [PATCH 0760/1072] [caliper] work on the regression_report html --- projects/caliper/engine/kpi/html_generator.py | 12 +- projects/caliper/templates/base.html | 77 +- .../caliper/templates/regression_report.html | 917 ++++++++++++++++-- 3 files changed, 933 insertions(+), 73 deletions(-) diff --git a/projects/caliper/engine/kpi/html_generator.py b/projects/caliper/engine/kpi/html_generator.py index dbc98924b..4bb90105e 100644 --- a/projects/caliper/engine/kpi/html_generator.py +++ b/projects/caliper/engine/kpi/html_generator.py @@ -105,13 +105,23 @@ def generate_regression_html( else: data = report_data + # Filter regression results with relative change for visualization + results = data.get("results", []) + regression_data = [ + result + for result in results + if result.get("verdict") == "REGRESSION" + and result.get("details", {}).get("relative_change") is not None + ] + html_content = template.render( report=data, overall=data.get("overall", {}), tested=data.get("tested", {}), - results=data.get("results", []), + results=results, analysis=data.get("analysis", {}), config=data.get("config", {}), + regression_data=regression_data, ) output_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/projects/caliper/templates/base.html b/projects/caliper/templates/base.html index cce10fcfd..f889c1de8 100644 --- a/projects/caliper/templates/base.html +++ b/projects/caliper/templates/base.html @@ -199,6 +199,43 @@ border: 1px solid #ddd; } } + + /* Error styling */ + .error-message { + color: #721c24; + background-color: #f8d7da; + border: 1px solid #f5c6cb; + padding: 0.75rem; + border-radius: 0.25rem; + margin: 0.5rem 0; + } + + .console-error { + font-family: monospace; + font-size: 0.875rem; + white-space: pre-wrap; + border-left: 4px solid #dc3545; + padding-left: 1rem; + } + + /* Console error override for better visibility */ + .console-error-display { + position: fixed; + top: 20px; + right: 20px; + max-width: 400px; + z-index: 9999; + color: #721c24; + background-color: #f8d7da; + border: 2px solid #f5c6cb; + padding: 1rem; + border-radius: 0.5rem; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2); + font-family: monospace; + font-size: 0.875rem; + max-height: 300px; + overflow-y: auto; + } {% block extra_head %}{% endblock %} @@ -215,5 +252,43 @@

{% block header_title %}Caliper Report{% endblock %}

{% block extra_js %}{% endblock %} + + - \ No newline at end of file + diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index 9132401ee..5465c0952 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -82,7 +82,6 @@

Regressions Detected ({{ regression_results|length }})

KPI Current Value - Baseline Mean Change Algorithm Details @@ -100,8 +99,8 @@

Regressions Detected ({{ regression_results|length }})

{% if result.current_value %} {% if result.is_curve %} - {% if result.current_value.values %} - {{ result.current_value.values|length }} points + {% if result.current_value['values'] %} + {{ result.current_value['values']|length }} points {% else %} No data {% endif %} @@ -116,13 +115,6 @@

Regressions Detected ({{ regression_results|length }})

No data {% endif %} - - {% if result.details and result.details.baseline_mean is defined %} - {{ "%.3f"|format(result.details.baseline_mean) if result.details.baseline_mean is number else result.details.baseline_mean }} - {% else %} - - - {% endif %} - {% if result.details and result.details.relative_change is defined %} @@ -158,14 +150,70 @@

Regressions Detected ({{ regression_results|length }})

All Test Results

- + +
+
Filters
+ + +
+ +
+ +
+
+ +
+ +
+
+
+ + +
+ +
+
+ +
+ +
+
+
+ + +
+ +
+
+ +
+ +
+
+
+
+ + +
+ +
+
+ +
- - + + + @@ -183,25 +231,25 @@

All Test Results

+ - + + + + + + + + + + + + + + + {% endfor %}
KPI VerdictCurrent ValueBaselinesRelative ChangeDirection LabelsDetails
- {% if result.current_value %} - {% if result.is_curve %} - {% if result.current_value.values %} - {{ result.current_value.values|length }} points - {% else %} - No data - {% endif %} + {% if result.details and result.details.relative_change is not none %} + + {{ "%+.2f%%"|format(result.details.relative_change * 100) }} + + {% else %} + - + {% endif %} + + {% if result.higher_is_better is defined %} + {% if result.higher_is_better %} + ↑ Higher better {% else %} - {% if result.current_value.value is not none %} - {{ "%.3f"|format(result.current_value.value) if result.current_value.value is number else result.current_value.value }} - {% else %} - No data - {% endif %} + ↓ Lower better {% endif %} {% else %} - No data + - {% endif %} {{ result.baseline_count }} {% if result.labels and result.labels.comparison_keys %}
@@ -213,22 +261,230 @@

All Test Results

- {% endif %}
+
+ + + +
+
- -{% set regression_data = results | selectattr('verdict', 'equalto', 'REGRESSION') | selectattr('details.relative_change', 'defined') | list %} -{% if regression_data %} -
-

Regression Trend Visualization

-
-
-
-
-{% endif %} + {% if not results %}
@@ -239,50 +495,157 @@

No Results Available

{% endblock %} {% block extra_js %} -{% if regression_data %} + + +{% set curve_kpis = results | selectattr('is_curve', 'equalto', true) | list %} +{% set scalar_kpis = results | selectattr('is_curve', 'equalto', false) | list %} + + + -{% endif %} -{% endblock %} \ No newline at end of file + +{% endblock %} From 2acf90446f25d47b859bc295cb4069872e8cf1a3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 14:10:18 +0200 Subject: [PATCH 0761/1072] [llm_d] postprocess: llm_d: parsing/kpis: sort the curved KPIs by X value --- .../llm_d/postprocess/llm_d/parsing/kpis.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/projects/llm_d/postprocess/llm_d/parsing/kpis.py b/projects/llm_d/postprocess/llm_d/parsing/kpis.py index d24b564e7..2b2bf48f9 100644 --- a/projects/llm_d/postprocess/llm_d/parsing/kpis.py +++ b/projects/llm_d/postprocess/llm_d/parsing/kpis.py @@ -190,13 +190,16 @@ def compute_kpis( if not is_curve_kpi(kpi_func): continue - try: - # Pass the single record with performance curves to the curve KPI function - raw_values = kpi_func(r) - # Convert list of tuples to list of lists for schema compatibility - values = [[float(x), float(y)] for x, y in raw_values] if raw_values else [] - except (TypeError, ValueError, KeyError): - values = [] # Empty list for failed curve KPIs + # Pass the single record with performance curves to the curve KPI function + raw_values = kpi_func(r) + # Convert list of tuples to list of lists for schema compatibility and sort by X value + # Preserve original numeric types (int/float) instead of forcing conversion to float + if raw_values: + coordinate_pairs = [[x, y] for x, y in raw_values] + # Sort by X value (first element of each coordinate pair) + values = sorted(coordinate_pairs, key=lambda point: point[0]) + else: + values = [] # Skip curve KPIs with empty or null values if not values or values is None: From ce334cc8ee96ff7e878c43ea8334562acda77e33 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 15:02:30 +0200 Subject: [PATCH 0762/1072] [caliper] engine: kpi: analyze: track the total and same_version count for data sources --- projects/caliper/engine/kpi/analyze.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index d58c85452..b787d0528 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -549,6 +549,9 @@ def _unique_values(key: str) -> list[str]: seen_all = [] seen_filtered = [] + same_version_count = 0 + total_count = len(all_labels) + for labels in all_labels: # Use unified filtering logic filtered = _filter_labels_for_matching(labels, set(config.ignored_labels)) @@ -564,6 +567,7 @@ def _unique_values(key: str) -> list[str]: (k, str(labels[k])) for k in config.comparison_labels if k in labels ) if baseline_comparison_keys in current_comparison_combinations: + same_version_count += 1 continue if filtered and filtered not in seen_filtered: @@ -602,6 +606,8 @@ def _unique_values(key: str) -> list[str]: "relevant_distinct_labels": distinct_labels, "relevant_count": len(seen_filtered), "irrelevant_count": len(seen_all) - len(seen_filtered), + "same_version_count": same_version_count, + "total_count": total_count, } From 14d9617e961e7e892a945ed85730f60496f3a740 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 15:04:33 +0200 Subject: [PATCH 0763/1072] [caliper] engine: kpi: add a dataclass for LabelSetSummary --- projects/caliper/engine/kpi/analyze.py | 33 +++++++++---------- .../caliper/engine/kpi/report_dataclasses.py | 28 ++++++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index b787d0528..eecdde903 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -21,6 +21,7 @@ AnalysisSummary, CurrentValueInfo, InputDataSection, + LabelSetSummary, OverallSection, OverallStatus, RegressionReport, @@ -521,7 +522,7 @@ def _summarize_label_sets( config: AnalysisConfig, current_keys: dict[str, set[str]] | None = None, current_comparison_combinations: set[frozenset] | None = None, -) -> dict[str, Any]: +) -> LabelSetSummary: """Summarize label sets found in a hierarchical KPI doc. Args: @@ -594,21 +595,19 @@ def _unique_values(key: str) -> list[str]: else: common, distinct_keys, distinct_labels = [], [], [] - return { - "comparison_keys": sorted( + return LabelSetSummary( + comparison_keys=sorted( f"{k}={v}" for k in config.comparison_labels for v in _unique_values(k) ), - "ignored_keys": sorted( - f"{k}={v}" for k in config.ignored_labels for v in _unique_values(k) - ), - "relevant_common_keys": common, - "relevant_distinct_keys": distinct_keys, - "relevant_distinct_labels": distinct_labels, - "relevant_count": len(seen_filtered), - "irrelevant_count": len(seen_all) - len(seen_filtered), - "same_version_count": same_version_count, - "total_count": total_count, - } + ignored_keys=sorted(f"{k}={v}" for k in config.ignored_labels for v in _unique_values(k)), + relevant_common_keys=common, + relevant_distinct_keys=distinct_keys, + relevant_distinct_labels=distinct_labels, + relevant_count=len(seen_filtered), + irrelevant_count=len(seen_all) - len(seen_filtered), + same_version_count=same_version_count, + total_count=total_count, + ) def _build_report( @@ -931,7 +930,7 @@ def run_kpi_analysis( source_entry = { "path": str(path), - **summary, + **summary.to_dict(), "unexpected_labels": sorted(unexpected_labels), "irrelevant_keys": sorted(irrelevant_keys), } @@ -942,14 +941,14 @@ def run_kpi_analysis( source_entry.pop("irrelevant_keys") # Split sources based on relevant_count - if summary.get("relevant_count", 0) > 0: + if summary.relevant_count > 0: relevant_sources.append(source_entry) else: irrelevant_sources.append(source_entry) current_source = { "path": str(current_kpi_file), - **_summarize_label_sets(current_data, config, None, None), + **_summarize_label_sets(current_data, config, None, None).to_dict(), } if (irr_count := current_source.pop("irrelevant_count")) != 0: diff --git a/projects/caliper/engine/kpi/report_dataclasses.py b/projects/caliper/engine/kpi/report_dataclasses.py index 0ad35e0e0..2d33eeb37 100644 --- a/projects/caliper/engine/kpi/report_dataclasses.py +++ b/projects/caliper/engine/kpi/report_dataclasses.py @@ -209,6 +209,34 @@ def from_dict(cls, data: dict[str, Any]) -> OverallSection: ) +@dataclass +class LabelSetSummary: + """Summary of label sets found in a hierarchical KPI document. + + Contains analysis of how many entries are relevant, irrelevant, or filtered + for various reasons when comparing against current test data. + """ + + comparison_keys: list[str] = field(default_factory=list) + ignored_keys: list[str] = field(default_factory=list) + relevant_common_keys: list[str] = field(default_factory=list) + relevant_distinct_keys: list[str] = field(default_factory=list) + relevant_distinct_labels: list[str] = field(default_factory=list) + relevant_count: int = 0 + irrelevant_count: int = 0 + same_version_count: int = 0 + total_count: int = 0 + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> LabelSetSummary: + """Create LabelSetSummary from dictionary data.""" + return cls(**data) + + @dataclass class InputDataSection: """Input data section of the regression report.""" From 202511a65680fc4b88aac574b277d1cd88ff964e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 15:05:12 +0200 Subject: [PATCH 0764/1072] [caliper] templates: regression_report: add a panel for the input files --- projects/caliper/engine/kpi/html_generator.py | 1 + .../caliper/templates/regression_report.html | 302 ++++++++++++++++++ 2 files changed, 303 insertions(+) diff --git a/projects/caliper/engine/kpi/html_generator.py b/projects/caliper/engine/kpi/html_generator.py index 4bb90105e..4d7a480ab 100644 --- a/projects/caliper/engine/kpi/html_generator.py +++ b/projects/caliper/engine/kpi/html_generator.py @@ -121,6 +121,7 @@ def generate_regression_html( results=results, analysis=data.get("analysis", {}), config=data.get("config", {}), + input_data=data.get("input_data", {}), regression_data=regression_data, ) diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index 5465c0952..485459f9a 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -492,6 +492,293 @@

No Results Available

No regression analysis results were found in the provided file.

{% endif %} + + +
+

Input Data Sources

+ + {% if input_data %} + + {% if input_data.current_source %} +

Current Data

+ + + {% for key, value in input_data.current_source.items() %} + + + + + {% endfor %} + +
{{ key|title|replace('_', ' ') }} + {% if key in ['file_path', 'path', 'source'] %} + {{ value }} + + {% elif key == 'comparison_keys' and value %} +
+ {% for item in value %} + {{ item }} + {% endfor %} +
+ + {% elif key == 'ignored_keys' and value %} +
+ {% for item in value %} + {{ item }} + {% endfor %} +
+ + {% elif key == 'relevant_common_keys' and value %} +
+ {% for item in value %} + {{ item }} + {% endfor %} +
+ + {% elif key == 'relevant_distinct_keys' and value %} +
+ {% for item in value %} + {% if item %} + {{ item }} + {% endif %} + {% endfor %} +
+ + {% elif key == 'relevant_distinct_labels' and value %} +
+ {% for item in value %} + {% if item %} + {{ item }} + {% endif %} + {% endfor %} +
+ + {% elif key in ['relevant_count', 'irrelevant_count'] %} + {{ value }} + + {% elif key in ['timestamp', 'modified', 'created'] %} + {{ value }} + {% elif key in ['size', 'file_size'] %} + {{ value }} + {% elif key in ['type', 'format'] %} + {{ value }} + {% else %} + {{ value }} + {% endif %} +
+ {% endif %} + + + {% if input_data.baseline_sources %} +

Baseline Data Sources

+ + + {% if input_data.baseline_sources.relevant_sources %} +

+ ▶ ✓ Relevant Sources ({{ input_data.baseline_sources.relevant_sources|length }}) +

+ + {% endif %} + + + {% if input_data.baseline_sources.irrelevant_sources %} +

+ ▶ ✗ Irrelevant Sources ({{ input_data.baseline_sources.irrelevant_sources|length }}) +

+ +
+ {% endif %} + {% endif %} + + {% else %} +
+

Input data source information not available

+ This information is populated by the regression analysis engine +
+ {% endif %} +
{% endblock %} {% block extra_js %} @@ -771,6 +1058,7 @@

No Results Available

{ kpi_id: '{{ result.kpi_id }}', verdict: '{{ result.verdict }}', + run_id: '{{ result.run_id }}', is_curve: {{ result.is_curve | tojson }}, current_value: {{ result.current_value | tojson | safe }}, baseline_values: {{ result.baseline_values | tojson | safe }}, @@ -1069,6 +1357,20 @@

No Results Available

}, 100); }); +// Toggle function for collapsible source sections +function toggleSourceSection(sectionId) { + const content = document.getElementById(`${sectionId}-content`); + const icon = document.getElementById(`${sectionId}-icon`); + + if (content.style.display === 'none' || content.style.display === '') { + content.style.display = 'block'; + icon.textContent = '▼'; + } else { + content.style.display = 'none'; + icon.textContent = '▶'; + } +} + {% endblock %} From bc6756609ed07f741a15fa357e4fe840a32f45e7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 15:21:55 +0200 Subject: [PATCH 0765/1072] [caliper] templates: regression_report: Allow jumping from regression to KPI details --- .../caliper/templates/regression_report.html | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index 485459f9a..1522684c0 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -91,7 +91,7 @@

Regressions Detected ({{ regression_results|length }})

{% for result in regression_results %} - {{ result.kpi_id }} + {{ result.kpi_id }} {% if result.run_id %}
{{ result.run_id }} {% endif %} @@ -201,7 +201,7 @@
Filters
- +
@@ -218,7 +218,7 @@
Filters
{% for result in results %} - + {{ result.kpi_id }} {% if result.run_id %} @@ -1371,6 +1371,62 @@

Analysis Details

Curve AUC Analysis
- {% if result.current_value and result.current_value['values'] %} -
- Current AUC: - Calculating... -
- {% endif %} - - {% if result.current_value and result.current_value['values'] %} -
- AUC Change: - Calculating... -
- {% endif %} - - {% if result.baseline_values %} -
- Historical AUC Values: - - - - - - - - - - - - -
BaselineAUC Value
Calculating...
+ + + + + + + + + + + + + + + + {% if result.details.baseline_aucs %} + {% for baseline in result.details.baseline_aucs %} + + + + + {% endfor %} + + + + + + + {% endif %} + +
VersionAUC Value
+ Current + {% if result.current_value.comparison_keys %} +
+ {% for key, value in result.current_value.comparison_keys.items() %} + {{ key }}={{ value }}{% if not loop.last %}, {% endif %} + {% endfor %} + + {% endif %} + 		+ {{ "%.3f"|format(result.details.current_auc) }} +
+ {% if baseline.comparison_keys %} + {% for key, value in baseline.comparison_keys.items() %} + {{ key }}={{ value }}{% if not loop.last %}, {% endif %} + {% endfor %} + {% else %} + Historical {{ loop.index }} + {% endif %} + + {{ "%.3f"|format(baseline.value) }} +
+ Baseline Mean + + {{ "%.3f"|format(result.details.baseline_mean_auc) }} +
+ + +
+
+
+ Relative Change: + + {{ "%+.1f%%"|format(result.details.relative_change) }} + +
+ {% if result.details.config and result.details.config.max_relative_regression %} +
+ Regression Threshold: + {{ "%+.1f%%"|format(result.details.config.max_relative_regression * 100) }} +
+ {% endif %} +
- {% endif %}
{% elif result.details.algorithm == 'SCALAR_RELATIVE_CHANGE' %} @@ -1069,103 +1111,6 @@

${label} - ${auc.toFixed(3)} - `; - tbody.appendChild(row); - } - }); - - // If no data, show placeholder - if (tbody.children.length === 0) { - tbody.innerHTML = 'No historical data available'; - } - } - } - - // Calculate AUC change - if (result.current_value && result.current_value.values && historicalAUCs.length > 0) { - const currentAUC = calculateAUC(result.current_value.values); - const avgHistoricalAUC = historicalAUCs.reduce((sum, auc) => sum + auc, 0) / historicalAUCs.length; - const aucChange = currentAUC - avgHistoricalAUC; - const aucChangePercent = avgHistoricalAUC !== 0 ? (aucChange / avgHistoricalAUC) * 100 : 0; - - const aucChangeElement = document.getElementById(`auc-change-${index}`); - if (aucChangeElement) { - const sign = aucChange >= 0 ? '+' : ''; - aucChangeElement.innerHTML = `${sign}${aucChange.toFixed(3)} (${sign}${aucChangePercent.toFixed(2)}%)`; - aucChangeElement.style.color = aucChange >= 0 ? '#dc3545' : '#28a745'; - } - } - } - } - }); -} // Filter functionality function populateFilterOptions() { @@ -1348,12 +1293,11 @@

Date: Tue, 8 Sep 2026 16:22:10 +0200 Subject: [PATCH 0768/1072] [caliper] engine: kpi: analyze: set the x/y unit/help fields --- projects/caliper/engine/kpi/analyze.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index eecdde903..47133cce3 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -346,6 +346,12 @@ def _run_regression_test( ), baseline_values=baseline_values_list, baseline_count=len(baseline_values_list), + unit=current.unit, + help=current.help, + x_unit=current.x_unit, + x_help=current.x_help, + y_unit=current.y_unit, + y_help=current.y_help, ) if is_curve: @@ -700,6 +706,12 @@ def _build_report( baseline_values=result.baseline_values, baseline_count=result.baseline_count, details=details, + unit=result.unit, + help=result.help, + x_unit=result.x_unit, + x_help=result.x_help, + y_unit=result.y_unit, + y_help=result.y_help, ) result_entries.append(entry) From aa8744925fbbb35d4866e8c1fbc0a233867b5c5d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 16:22:30 +0200 Subject: [PATCH 0769/1072] [caliper] engine: kpi: report_dataclasses: define the x/y help/unit fields --- projects/caliper/engine/kpi/report_dataclasses.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/projects/caliper/engine/kpi/report_dataclasses.py b/projects/caliper/engine/kpi/report_dataclasses.py index 2d33eeb37..1d966ee6b 100644 --- a/projects/caliper/engine/kpi/report_dataclasses.py +++ b/projects/caliper/engine/kpi/report_dataclasses.py @@ -114,6 +114,12 @@ class ResultEntry: baseline_values: list[dict[str, Any]] = field(default_factory=list) baseline_count: int = 0 details: dict[str, Any] = field(default_factory=dict) + unit: str = "" + help: str = "" + x_unit: str = "" + x_help: str = "" + y_unit: str = "" + y_help: str = "" def to_dict(self) -> dict[str, Any]: """Convert to dictionary for JSON serialization.""" @@ -324,6 +330,12 @@ class RegressionTestResult: baseline_count: int = 0 reason: str | None = None details: dict[str, Any] = field(default_factory=dict) + unit: str = "" + help: str = "" + x_unit: str = "" + x_help: str = "" + y_unit: str = "" + y_help: str = "" def to_dict(self) -> dict[str, Any]: """Convert to dictionary for JSON serialization.""" From 8d7a121c8d22d2b8f8696e7088a0429e172f0d19 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 17:05:25 +0200 Subject: [PATCH 0770/1072] [caliper] include Verdict.NO_BASELINE for regression analyze --- projects/caliper/engine/kpi/analyze.py | 17 +++++++++++------ .../caliper/engine/kpi/report_dataclasses.py | 1 + .../caliper/templates/regression_report.html | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 47133cce3..df39be1b3 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -75,7 +75,7 @@ def create_analysis_summary( passes = [r for r in results if r.verdict == Verdict.PASS] regressions = [r for r in results if r.verdict == Verdict.REGRESSION] - skipped = [r for r in results if r.verdict == Verdict.SKIPPED] + skipped = [r for r in results if r.verdict in (Verdict.SKIPPED, Verdict.NO_BASELINE)] test_summary = TestSummary( total_kpis=len(results), @@ -385,7 +385,7 @@ def _scalar_relative_change_regression( return base if len(scalar_entries) < min_baseline_points: - base.verdict = Verdict.SKIPPED + base.verdict = Verdict.NO_BASELINE base.reason = f"insufficient baselines ({len(scalar_entries)} < {min_baseline_points})" return base @@ -441,7 +441,7 @@ def _curve_auc_change_regression( ] if len(auc_baselines) < min_baseline_points: - base.verdict = Verdict.SKIPPED + base.verdict = Verdict.NO_BASELINE base.reason = f"insufficient curve baselines ({len(auc_baselines)} < {min_baseline_points})" return base @@ -512,9 +512,14 @@ def _sort_results( results: list[RegressionTestResult], sorting_labels: list[str] ) -> list[RegressionTestResult]: """Sort results by sorting labels extracted from labels, then by kpi_id. - SKIPPED entries are placed after tested entries.""" + SKIPPED and NO_BASELINE entries are placed after tested entries.""" - verdict_order = {Verdict.PASS: 0, Verdict.REGRESSION: 1, Verdict.SKIPPED: 2} + verdict_order = { + Verdict.PASS: 0, + Verdict.REGRESSION: 1, + Verdict.SKIPPED: 2, + Verdict.NO_BASELINE: 2, + } def sort_key(r: RegressionTestResult) -> tuple: label_key = tuple(str(r.labels.get(k, "")) for k in sorting_labels) @@ -627,7 +632,7 @@ def _build_report( """Build the final report structure using original format.""" regressions = [r for r in results if r.verdict == Verdict.REGRESSION] passes = [r for r in results if r.verdict == Verdict.PASS] - skipped = [r for r in results if r.verdict == Verdict.SKIPPED] + skipped = [r for r in results if r.verdict in (Verdict.SKIPPED, Verdict.NO_BASELINE)] if regressions: overall_status = OverallStatus.REGRESSION_DETECTED diff --git a/projects/caliper/engine/kpi/report_dataclasses.py b/projects/caliper/engine/kpi/report_dataclasses.py index 1d966ee6b..8cdb6b095 100644 --- a/projects/caliper/engine/kpi/report_dataclasses.py +++ b/projects/caliper/engine/kpi/report_dataclasses.py @@ -33,6 +33,7 @@ class Verdict(StrEnum): PASS = "PASS" REGRESSION = "REGRESSION" SKIPPED = "SKIPPED" + NO_BASELINE = "NO_BASELINE" @dataclass diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index cf74f4c2b..2a450bdc0 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -226,7 +226,7 @@
Filters
{% endif %} - + {{ result.verdict }} @@ -1154,7 +1154,7 @@

${verdict}`; From ff532f1acced16b998ed20d5812c68289aaf7640 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 17:08:33 +0200 Subject: [PATCH 0771/1072] [caliper] remove the same_version field --- projects/caliper/engine/kpi/analyze.py | 22 +++++++++++-------- .../caliper/engine/kpi/report_dataclasses.py | 2 +- projects/caliper/templates/kpi_dashboard.html | 8 +++---- .../caliper/templates/regression_report.html | 14 +----------- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index df39be1b3..9d268a827 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -561,7 +561,7 @@ def _unique_values(key: str) -> list[str]: seen_all = [] seen_filtered = [] - same_version_count = 0 + irrelevant_keys = [] total_count = len(all_labels) for labels in all_labels: @@ -579,7 +579,11 @@ def _unique_values(key: str) -> list[str]: (k, str(labels[k])) for k in config.comparison_labels if k in labels ) if baseline_comparison_keys in current_comparison_combinations: - same_version_count += 1 + # Add comparison keys to irrelevant_keys instead of counting + for k, v in baseline_comparison_keys: + key_str = f"{k}={v}" + if key_str not in irrelevant_keys: + irrelevant_keys.append(key_str) continue if filtered and filtered not in seen_filtered: @@ -614,9 +618,9 @@ def _unique_values(key: str) -> list[str]: relevant_common_keys=common, relevant_distinct_keys=distinct_keys, relevant_distinct_labels=distinct_labels, + irrelevant_keys=sorted(irrelevant_keys), relevant_count=len(seen_filtered), irrelevant_count=len(seen_all) - len(seen_filtered), - same_version_count=same_version_count, total_count=total_count, ) @@ -890,7 +894,7 @@ def run_kpi_analysis( # Run regression tests results: list[RegressionTestResult] = [] - baseline_skipped_totals: dict[str, int] = {"same_version": 0, "duplicate": 0} + baseline_skipped_totals: dict[str, int] = {"duplicate": 0} for test in current_data.tests: test_labels = test.labels @@ -907,7 +911,6 @@ def run_kpi_analysis( for ck, baseline_entry in baseline_dict.items() if ck != current_ck ] - baseline_skipped_totals["same_version"] += len(baseline_dict) - len(baselines) if not baselines: _log_baseline_miss(kpi.kpi_id, mk, baseline_index) @@ -945,16 +948,19 @@ def run_kpi_analysis( entry = f"{k}={sv}" irrelevant_keys.add(entry) + # Merge irrelevant_keys from summary (includes comparison keys) with local irrelevant_keys + all_irrelevant_keys = set(summary.irrelevant_keys) | irrelevant_keys + source_entry = { "path": str(path), **summary.to_dict(), "unexpected_labels": sorted(unexpected_labels), - "irrelevant_keys": sorted(irrelevant_keys), + "irrelevant_keys": sorted(all_irrelevant_keys), } if not unexpected_labels: source_entry.pop("unexpected_labels") - if not irrelevant_keys: + if not all_irrelevant_keys: source_entry.pop("irrelevant_keys") # Split sources based on relevant_count @@ -971,8 +977,6 @@ def run_kpi_analysis( if (irr_count := current_source.pop("irrelevant_count")) != 0: logger.error(f"Found {irr_count} irrelevant entries in the current_source. Expected 0.") logger.info(f"Found {len(relevant_sources)} relevant files") - if not baseline_skipped_totals["same_version"]: - baseline_skipped_totals.pop("same_version") if not baseline_skipped_totals["duplicate"]: baseline_skipped_totals.pop("duplicate") diff --git a/projects/caliper/engine/kpi/report_dataclasses.py b/projects/caliper/engine/kpi/report_dataclasses.py index 8cdb6b095..bd7765af5 100644 --- a/projects/caliper/engine/kpi/report_dataclasses.py +++ b/projects/caliper/engine/kpi/report_dataclasses.py @@ -229,9 +229,9 @@ class LabelSetSummary: relevant_common_keys: list[str] = field(default_factory=list) relevant_distinct_keys: list[str] = field(default_factory=list) relevant_distinct_labels: list[str] = field(default_factory=list) + irrelevant_keys: list[str] = field(default_factory=list) relevant_count: int = 0 irrelevant_count: int = 0 - same_version_count: int = 0 total_count: int = 0 def to_dict(self) -> dict[str, Any]: diff --git a/projects/caliper/templates/kpi_dashboard.html b/projects/caliper/templates/kpi_dashboard.html index a0cb123f6..85fad28bb 100644 --- a/projects/caliper/templates/kpi_dashboard.html +++ b/projects/caliper/templates/kpi_dashboard.html @@ -175,18 +175,18 @@
{{ kpi.name or kpi.kpi_id }}

Curve Visualizations

{% for kpi_id in curve_kpi_ids %} - {% set first_kpi = none %} + {% set ns = namespace(first_kpi=none) %} {% for test in tests %} {% for kpi in test.kpis %} {% if kpi.kpi_id == kpi_id and kpi.is_curve %} - {% if not first_kpi %} - {% set first_kpi = kpi %} + {% if not ns.first_kpi %} + {% set ns.first_kpi = kpi %} {% endif %} {% endif %} {% endfor %} {% endfor %}
-

{{ first_kpi.name or kpi_id if first_kpi else kpi_id }}

+

{{ ns.first_kpi.name or kpi_id if ns.first_kpi else kpi_id }}

diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index 2a450bdc0..a89cc8da7 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -489,7 +489,7 @@
Curve AU
Relative Change: - {{ "%+.1f%%"|format(result.details.relative_change) }} + {{ "%+.1f%%"|format(result.details.relative_change * 100) }}
{% if result.details.config and result.details.config.max_relative_regression %} @@ -651,12 +651,6 @@

{{ source.relevant_count }}

- {% if source.same_version_count is defined %} -
- Same Version Count: - {{ source.same_version_count }} -
- {% endif %} {% if source.total_count is defined %}
@@ -743,12 +737,6 @@

{{ source.irrelevant_count }}

- {% if source.same_version_count is defined %} -
- Same Version Count: - {{ source.same_version_count }} -
- {% endif %} {% if source.total_count is defined %}
From 39e6bf971c747b51c5b9c77f51f3dcff9aa4c578 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 17:21:49 +0200 Subject: [PATCH 0772/1072] [llm_d] postprocess: llm_d: parsing/test_kpis: new test --- .../postprocess/llm_d/parsing/test_kpis.py | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 projects/llm_d/postprocess/llm_d/parsing/test_kpis.py diff --git a/projects/llm_d/postprocess/llm_d/parsing/test_kpis.py b/projects/llm_d/postprocess/llm_d/parsing/test_kpis.py new file mode 100644 index 000000000..7150093cb --- /dev/null +++ b/projects/llm_d/postprocess/llm_d/parsing/test_kpis.py @@ -0,0 +1,221 @@ +"""Tests for KPI parsing functionality.""" + +from __future__ import annotations + +import json +import unittest +from unittest.mock import Mock, patch + +from projects.caliper.engine.model import UnifiedRunModel +from projects.llm_d.postprocess.llm_d.parsing.kpis import GuideLLMKpiHandler + + +class TestGuideLLMKpiHandler(unittest.TestCase): + """Test cases for GuideLLM KPI handler coordinate conversion.""" + + def test_coordinate_conversion_preserves_integer_types(self): + """Test that coordinate conversion preserves original numeric types without unnecessary float conversion.""" + + # Create mock data with large integer X values and mixed numeric types + mock_record = Mock() + mock_record.test_base_path = "test_path" + mock_record.metrics = { + "performance_curves": {"latency": [(1000000, 0.5), (2000000, 1.0), (3000000, 1.5)]}, + "request_rate": [100, 200, 300], + "product_version": "v1.0", + "cluster": "test-cluster", + "model_name": "test-model", + "benchmark_key": "test", + } + + # Create a mock curve KPI function that returns various numeric types + def mock_curve_kpi_func(record): + """Mock curve KPI function that returns coordinates with large integers and floats.""" + return [ + (1000000, 0.5), # Large int X, float Y + (3000000, 1.5), # Large int X, float Y + (2000000, 1), # Large int X, int Y + (500000, 0.25), # Medium int X, float Y + (4000000, 2.0), # Large int X, float Y + ] + + # Add KPI metadata attributes to the mock function + mock_curve_kpi_func._kpi_unit = "ms" + mock_curve_kpi_func._kpi_x_unit = "req/s" + mock_curve_kpi_func._kpi_x_help = "Request rate" + mock_curve_kpi_func._kpi_y_unit = "ms" + mock_curve_kpi_func._kpi_y_help = "Latency" + + # Create unified model with mock record + model = Mock(spec=UnifiedRunModel) + model.unified_result_records = [mock_record] + + # Mock the KPI functions and dependencies + mock_kpi_functions = {"test_latency": mock_curve_kpi_func} + + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.get_kpi_functions" + ) as mock_get_kpi_functions: + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.is_curve_kpi" + ) as mock_is_curve_kpi: + mock_get_kpi_functions.return_value = mock_kpi_functions + mock_is_curve_kpi.return_value = True + + # Call the compute_kpis method + kpi_records, status = GuideLLMKpiHandler.compute_kpis(model) + + # Verify we got a KPI record + assert len(kpi_records) == 1 + kpi_record = kpi_records[0] + + # Verify the coordinate types are preserved + values = kpi_record.values + assert len(values) == 5 + + # Verify the values are sorted by X coordinate + expected_sorted_values = [ + [500000, 0.25], # Sorted by X value + [1000000, 0.5], + [2000000, 1], # Integer Y value preserved + [3000000, 1.5], + [4000000, 2.0], + ] + + assert values == expected_sorted_values + + # Verify that integer types are preserved (not converted to float) + # The large integer X values should remain as integers + assert isinstance(values[0][0], int) # 500000 + assert isinstance(values[1][0], int) # 1000000 + assert isinstance(values[2][0], int) # 2000000 + assert isinstance(values[2][1], int) # 1 (integer Y value preserved) + assert isinstance(values[3][0], int) # 3000000 + assert isinstance(values[4][0], int) # 4000000 + + # Verify that float types are preserved + assert isinstance(values[0][1], float) # 0.25 + assert isinstance(values[1][1], float) # 0.5 + assert isinstance(values[3][1], float) # 1.5 + assert isinstance(values[4][1], float) # 2.0 + + # Verify serialization preserves types + serialized = json.dumps(values) + deserialized = json.loads(serialized) + + # After JSON round-trip, integers should remain integers (as long as they're within safe range) + # Note: JSON preserves integers up to JavaScript's safe integer limit + for i, (x, y) in enumerate(deserialized): + original_x, original_y = values[i] + # Large integers should be preserved in JSON + assert x == original_x + assert y == original_y + + def test_coordinate_conversion_handles_empty_values(self): + """Test that coordinate conversion handles empty or null values correctly.""" + + mock_record = Mock() + mock_record.test_base_path = "test_path" + mock_record.metrics = { + "performance_curves": {"latency": []}, + "request_rate": [100], + "product_version": "v1.0", + "cluster": "test-cluster", + "model_name": "test-model", + "benchmark_key": "test", + } + + def mock_empty_kpi_func(record): + """Mock KPI function that returns empty values.""" + return [] + + mock_empty_kpi_func._kpi_unit = "ms" + mock_empty_kpi_func._kpi_x_unit = "req/s" + mock_empty_kpi_func._kpi_x_help = "Request rate" + mock_empty_kpi_func._kpi_y_unit = "ms" + mock_empty_kpi_func._kpi_y_help = "Latency" + + model = Mock(spec=UnifiedRunModel) + model.unified_result_records = [mock_record] + + mock_kpi_functions = {"test_empty": mock_empty_kpi_func} + + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.get_kpi_functions" + ) as mock_get_kpi_functions: + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.is_curve_kpi" + ) as mock_is_curve_kpi: + mock_get_kpi_functions.return_value = mock_kpi_functions + mock_is_curve_kpi.return_value = True + + kpi_records, status = GuideLLMKpiHandler.compute_kpis(model) + + # Empty values should result in no KPI records being created + assert len(kpi_records) == 0 + + def test_coordinate_sorting_with_large_integers(self): + """Test that coordinate sorting works correctly with large integer values.""" + + mock_record = Mock() + mock_record.test_base_path = "test_path" + mock_record.metrics = { + "performance_curves": {"latency": []}, + "request_rate": [100], + "product_version": "v1.0", + "cluster": "test-cluster", + "model_name": "test-model", + "benchmark_key": "test", + } + + def mock_unsorted_kpi_func(record): + """Mock KPI function with unsorted large integer coordinates.""" + return [ + (9999999999, 5.0), # Very large X value + (1, 1.0), # Small X value + (1000000000, 3.0), # Large X value + (500, 2.0), # Medium X value + ] + + mock_unsorted_kpi_func._kpi_unit = "ms" + mock_unsorted_kpi_func._kpi_x_unit = "req/s" + mock_unsorted_kpi_func._kpi_x_help = "Request rate" + mock_unsorted_kpi_func._kpi_y_unit = "ms" + mock_unsorted_kpi_func._kpi_y_help = "Latency" + + model = Mock(spec=UnifiedRunModel) + model.unified_result_records = [mock_record] + + mock_kpi_functions = {"test_sort": mock_unsorted_kpi_func} + + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.get_kpi_functions" + ) as mock_get_kpi_functions: + with patch( + "projects.llm_d.postprocess.llm_d.parsing.kpis.is_curve_kpi" + ) as mock_is_curve_kpi: + mock_get_kpi_functions.return_value = mock_kpi_functions + mock_is_curve_kpi.return_value = True + + kpi_records, status = GuideLLMKpiHandler.compute_kpis(model) + + assert len(kpi_records) == 1 + values = kpi_records[0].values + + # Verify coordinates are sorted by X value (first element) + expected_sorted = [ + [1, 1.0], + [500, 2.0], + [1000000000, 3.0], + [9999999999, 5.0], + ] + + assert values == expected_sorted + + # Verify large integers are preserved as integers + assert isinstance(values[2][0], int) # 1000000000 + assert isinstance(values[3][0], int) # 9999999999 + + # Verify the large integer values are exactly what we expect + assert values[2][0] == 1000000000 + assert values[3][0] == 9999999999 From c68c05e810dbf1df3224f154f7857202fd9a98bf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 17:49:43 +0200 Subject: [PATCH 0773/1072] [caliper] include the html file in the status --- projects/caliper/cli/commands.py | 5 ++++- projects/caliper/engine/kpi/html_generator.py | 2 +- projects/caliper/orchestration/notification.py | 11 +++++++++++ projects/caliper/public/status_models.py | 2 ++ .../{kpi_dashboard.html => kpi_listing.html} | 12 ++++++------ projects/caliper/templates/regression_report.html | 1 - 6 files changed, 24 insertions(+), 9 deletions(-) rename projects/caliper/templates/{kpi_dashboard.html => kpi_listing.html} (94%) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 487789987..266d2fb2c 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -737,7 +737,7 @@ def kpi_generate( html_output = output.with_suffix(".html") generate_kpi_html_from_file(output, html_output) click.echo(f"📄 Generated HTML report: {html_output}") - status_data["html_output_file"] = str(html_output) + status_data["html_file"] = str(html_output) except Exception as e: click.echo(f"⚠️ Failed to generate HTML report: {e}", err=True) status_data["html_generation_error"] = str(e) @@ -1067,6 +1067,9 @@ def analyse_kpis_cmd( html_output = output.with_suffix(".html") generate_regression_html_from_file(output, html_output) click.echo(f"📄 Generated HTML regression report: {html_output}") + + # Store HTML file path in status data for notifications + status_data.html_file = str(html_output) except Exception as e: click.echo(f"⚠️ Failed to generate HTML report: {e}", err=True) diff --git a/projects/caliper/engine/kpi/html_generator.py b/projects/caliper/engine/kpi/html_generator.py index 4d7a480ab..5cdf3c2f9 100644 --- a/projects/caliper/engine/kpi/html_generator.py +++ b/projects/caliper/engine/kpi/html_generator.py @@ -37,7 +37,7 @@ def __init__(self, template_dir: Path | None = None): def generate_kpi_html(self, kpi_data: dict | list, output_file: Path) -> None: """Generate HTML report from KPI data.""" try: - template = self.env.get_template("kpi_dashboard.html") + template = self.env.get_template("kpi_listing.html") # Handle both hierarchical (dict) and flat (list) formats if isinstance(kpi_data, dict): diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 074325425..03d916304 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -128,6 +128,12 @@ def _format_artifacts_to_kpis_step(step_data: dict, get_file_link: callable | No output_file = step_data.get("output_file") if output_file: lines.append(_create_file_link(output_file, "📄", get_file_link)) + + # Include HTML file if available + html_file = step_data.get("html_file") + if html_file: + lines.append(_create_file_link(html_file, "🌐", get_file_link)) + return lines @@ -254,6 +260,11 @@ def _format_analyse_kpis_step(step_data: dict, get_file_link: callable | None) - if baseline_files_count is not None: lines.append(f" - 📈 Baseline files analyzed: `{baseline_files_count}`") + # Include HTML file if available + html_file = step_data.get("html_file") + if html_file: + lines.append(_create_file_link(html_file, "🌐", get_file_link)) + return lines diff --git a/projects/caliper/public/status_models.py b/projects/caliper/public/status_models.py index 289d72e49..90357f0a8 100644 --- a/projects/caliper/public/status_models.py +++ b/projects/caliper/public/status_models.py @@ -80,6 +80,7 @@ class KpiGenerateStatus(BaseStatus): output_file: str | None = None kpi_count: int | None = None plugin_module: str | None = None + html_file: str | None = None @property def success(self) -> bool: @@ -95,6 +96,7 @@ class KpiAnalysisStatus(BaseStatus): exit_code: int = 0 output_file: str | None = None regressions_detected: bool = False + html_file: str | None = None # Additional analysis-specific fields regression_count: int | None = None diff --git a/projects/caliper/templates/kpi_dashboard.html b/projects/caliper/templates/kpi_listing.html similarity index 94% rename from projects/caliper/templates/kpi_dashboard.html rename to projects/caliper/templates/kpi_listing.html index 85fad28bb..5e165c53d 100644 --- a/projects/caliper/templates/kpi_dashboard.html +++ b/projects/caliper/templates/kpi_listing.html @@ -1,8 +1,8 @@ {% extends "base.html" %} -{% block title %}Caliper KPI Dashboard{% endblock %} +{% block title %}Caliper KPI Report{% endblock %} -{% block header_title %}KPI Dashboard{% endblock %} +{% block header_title %}KPI Report{% endblock %} {% block header_subtitle %}Performance Metrics Report - {{ total_tests }} test(s), {{ total_kpis }} KPI(s){% endblock %} @@ -220,7 +220,7 @@

No Data Available

{% for kpi in test.kpis %} {% if kpi.kpi_id == kpi_id and kpi.is_curve and kpi.values_list and kpi.values_list|length > 0 %} // Data for {{ test.run_id }} - const rawData_{{ test.run_id|replace('-', '_')|replace('.', '_') }} = {{ kpi.values_json }}; + let rawData = {{ kpi.values_list|tojson|safe }}; if (!firstKpi) { firstKpi = { @@ -233,9 +233,9 @@

No Data Available

}; } - if (Array.isArray(rawData_{{ test.run_id|replace('-', '_')|replace('.', '_') }}) && rawData_{{ test.run_id|replace('-', '_')|replace('.', '_') }}.length > 0) { - const x_values = rawData_{{ test.run_id|replace('-', '_')|replace('.', '_') }}.map(point => point[0]); - const y_values = rawData_{{ test.run_id|replace('-', '_')|replace('.', '_') }}.map(point => point[1]); + if (Array.isArray(rawData) && rawData.length > 0) { + let x_values = rawData.map(point => point[0]); + let y_values = rawData.map(point => point[1]); traces.push({ x: x_values, diff --git a/projects/caliper/templates/regression_report.html b/projects/caliper/templates/regression_report.html index a89cc8da7..6aa3c7000 100644 --- a/projects/caliper/templates/regression_report.html +++ b/projects/caliper/templates/regression_report.html @@ -798,7 +798,6 @@

None: status=StepStatus.DISABLED, completed_at=time.time(), reason="kpi.artifacts_to_kpis disabled", + html_file=None, ), ) return @@ -1252,6 +1265,7 @@ def _run_analyse_kpis_step(self, plugin_module: str) -> None: status=StepStatus.DISABLED, completed_at=time.time(), reason="analyze disabled", + html_file=None, ), ) return @@ -1271,6 +1285,7 @@ def _run_analyse_kpis_step(self, plugin_module: str) -> None: status=StepStatus.FAILED, completed_at=time.time(), error=f"Current KPI file not found: {current_kpis_path}", + html_file=None, ), ) self.analyze_failed = True @@ -1299,6 +1314,7 @@ def _run_analyse_kpis_step(self, plugin_module: str) -> None: regression_count=status.regression_count, total_kpis=status.total_kpis, log_file=status.log_file, + html_file=status.html_file, ) self._add_step("analyse_kpis", result, result.log_file) diff --git a/projects/caliper/public/postprocess_models.py b/projects/caliper/public/postprocess_models.py index 48315d62b..0dedbecf8 100644 --- a/projects/caliper/public/postprocess_models.py +++ b/projects/caliper/public/postprocess_models.py @@ -100,6 +100,7 @@ class KpiGenerateStepResult(BaseStepResult): output_file: str | None = None error: str | None = None + html_file: str | None = None @dataclass @@ -114,6 +115,7 @@ class KpiAnalysisStepResult(BaseStepResult): total_kpis: int | None = None error: str | None = None message: str | None = None + html_file: str | None = None @dataclass From c943ff11e83060f78085c2258cd7447c0a56788a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 21:01:46 +0200 Subject: [PATCH 0775/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: embed the plot in HTML --- .../guidellm/plotting/performance_analysis.py | 427 ++++++++++++++++-- 1 file changed, 389 insertions(+), 38 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index c6c66524c..cc198a1e4 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -57,6 +57,134 @@ def _image_to_base64(image_path: str | Path) -> str: return "" +def _read_html_content(html_path: str | Path) -> str: + """Read HTML file content for embedding. + + Args: + html_path: Path to the HTML file + + Returns: + HTML content string + """ + try: + logger.info(f"🔍 Reading HTML content from: {html_path}") + + # Check if file exists + if not Path(html_path).exists(): + logger.warning(f"❌ HTML file not found: {html_path}") + return "" + + with open(html_path, encoding="utf-8") as html_file: + content = html_file.read() + content_size_kb = len(content.encode("utf-8")) / 1024 + logger.info(f" 📄 Read {content_size_kb:.1f} KB from HTML file") + + # Import re for regex operations + import re + + # Extract body content to avoid HTML document conflicts but keep all scripts + body_match = re.search(r"]*>(.*?)", content, re.DOTALL | re.IGNORECASE) + if body_match: + body_content = body_match.group(1) + body_size_kb = len(body_content.encode("utf-8")) / 1024 + logger.info(f" ✅ Extracted body content: {body_size_kb:.1f} KB") + return body_content + else: + # If no body tags found, use full content + logger.warning(" ⚠️ No body tags found, using full content") + logger.info(f" 📄 Using full HTML content: {content_size_kb:.1f} KB") + return content + except Exception as e: + logger.warning(f"❌ Failed to read HTML content from {html_path}: {e}") + return "" + + +def _create_plot_with_optional_png_spoiler( + html_content: str, + png_path: str | Path | None, + plot_name: str, + output_dir: Path, + html_path: str | Path | None = None, +) -> str: + """Create HTML content with embedded plot and optional PNG spoiler. + + Args: + html_content: HTML content to embed directly + png_path: Relative path to PNG file (None if PNG generation failed) + plot_name: Name of the plot for alt text + output_dir: Base output directory to resolve PNG path + html_path: Relative path to HTML file (for large content linking) + + Returns: + HTML string with embedded plot and optional PNG spoiler + """ + # Embed directly as requested + result = f""" +
+ {html_content} +
""" + + # Add simple link to PNG version if it exists + if png_path is not None: + full_png_path = Path(output_dir) / png_path + if full_png_path.exists(): + result += f""" + """ + else: + logger.debug(f"PNG file not found for {plot_name} at {full_png_path}") + else: + logger.debug(f"PNG generation was skipped for {plot_name} (HTML-only mode)") + + return result + + +def embed_plot_for_report( + png_path: str | Path | None, html_path: str | Path, plot_name: str, output_dir: Path +) -> str: + """ + Centralized function to embed plots in reports with HTML content and optional PNG spoiler. + + This is the single point to modify if we want to change how plots are embedded across all reports. + Currently embeds HTML directly with an optional PNG spoiler, but can be easily modified to change + the embedding behavior project-wide. + + Args: + png_path: Relative path to PNG file (None if PNG generation failed) + html_path: Relative path to HTML file + plot_name: Name of the plot for alt text + output_dir: Base output directory to resolve paths + + Returns: + HTML string with embedded plot content + """ + logger.info(f"🔗 Embedding plot: {plot_name}") + logger.info(f" 📁 Output dir: {output_dir}") + logger.info(f" 🌐 HTML path: {html_path}") + logger.info(f" 🖼️ PNG path: {png_path}") + + # Resolve full HTML path + full_html_path = Path(output_dir) / html_path + logger.info(f" 📄 Full HTML path: {full_html_path}") + + # Read HTML content for direct embedding + plot_html_content = _read_html_content(full_html_path) + + if not plot_html_content.strip(): + logger.warning(f" ⚠️ Empty HTML content for {plot_name}") + return f"

⚠️ Could not load interactive plot for {plot_name}

" + + # Create plot with optional PNG spoiler + result = _create_plot_with_optional_png_spoiler( + plot_html_content, png_path, plot_name, output_dir, html_path + ) + + result_size_kb = len(result.encode("utf-8")) / 1024 + logger.info(f" ✅ Successfully embedded {plot_name} ({result_size_kb:.1f} KB)") + return result + + # Filesystem-unsafe characters for path sanitization _PATH_UNSAFE_CHARS = ["/", "\\", ":", "*", "?", "|", "<", ">", '"'] @@ -483,6 +611,24 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): logger.info("⚠️ No data available for TTFT analysis plot") return None + # Debug: Check TTFT data availability + if "ttft_median_ms" not in df.columns: + logger.warning("⚠️ TTFT column 'ttft_median_ms' not found in dataframe") + logger.info(f" Available columns: {list(df.columns)}") + return None + + ttft_data = df["ttft_median_ms"] + non_zero_count = (ttft_data > 0).sum() + logger.info( + f" TTFT data: {len(ttft_data)} total points, {non_zero_count} non-zero values" + ) + logger.info(f" TTFT range: {ttft_data.min():.1f} - {ttft_data.max():.1f} ms") + + if ttft_data.max() == 0: + logger.warning("⚠️ All TTFT values are zero - plot may appear empty") + if non_zero_count < 2: + logger.warning("⚠️ Insufficient non-zero TTFT data for meaningful plot") + title = f"TTFT vs Concurrency{title_context}
Lower is better" # Get ordered configuration list to maintain consistent legend order @@ -533,6 +679,37 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st logger.info("⚠️ No data available for token throughput percentiles plot") return None + # Debug: Check percentile data availability + percentile_cols = [ + "output_tokens_per_second_p10", + "output_tokens_per_second_p25", + "output_tokens_per_second_p50", + "output_tokens_per_second_p75", + "output_tokens_per_second_p90", + ] + missing_cols = [col for col in percentile_cols if col not in df.columns] + if missing_cols: + logger.warning(f"⚠️ Missing percentile columns: {missing_cols}") + + available_cols = [col for col in percentile_cols if col in df.columns] + if not available_cols: + logger.warning("⚠️ No percentile columns found - cannot create percentiles plot") + return None + + logger.info( + f" Available percentile columns: {len(available_cols)}/{len(percentile_cols)}" + ) + + # Check if any percentile data is non-zero + has_data = False + for col in available_cols: + if col in df.columns and (df[col] > 0).any(): + has_data = True + break + + if not has_data: + logger.warning("⚠️ All percentile values are zero - plot may appear empty") + title = f"Output Token Throughput Percentiles{title_context}
Higher is better" fig = go.Figure() @@ -966,16 +1143,25 @@ def generate_deployment_profile_report( # Save HTML version html_path = save_figure(fig, group_dir, filename, as_image=False) - if png_path and html_path: - # Store relative paths for linking + if html_path: + # Store relative paths for linking (PNG optional) + png_rel_path = ( + f"{report_dir_name}/{group_name}/{Path(png_path).name}" + if png_path + else None + ) group_plots.append( ( plot_name, - f"{report_dir_name}/{group_name}/{Path(png_path).name}", # PNG path + png_rel_path, # PNG path (can be None) f"{report_dir_name}/{group_name}/{Path(html_path).name}", # HTML path ) ) logger.info(f" ✅ {plot_name} saved for comparison group") + if not png_path: + logger.warning( + f" ⚠️ PNG version of {plot_name} could not be generated (HTML available)" + ) else: logger.info( f" ⚠️ {plot_name} could not be created for comparison group (no figure returned)" @@ -1011,6 +1197,7 @@ def generate_deployment_profile_report( {display_title} + - - -
-

{escape(report_title)}

- -
- Test Run: {escape(str(test_info["test_path"]))}
- Duration: {escape(str(test_info["duration"]))} | - Requests: {escape(str(test_info["successful_requests"]))}/{escape(str(test_info["total_requests"]))} successful -
- -

🏷️ Test Conditions

- - - - - - - - -""" - - # Add test condition labels - for label, value in sorted(test_labels.items()): - if label != "higher_is_better": # Skip internal labels - html += f""" - - - - -""" - - html += """ - -
LabelValue
{escape(label.replace("_", " ").title())}{escape(str(value))}
-""" - - # Add metadata section if there's any metadata - if metadata and any(v for v in metadata.values() if v): - html += """ -

📋 Test Metadata

- - - - - - - - -""" - - for key, value in metadata.items(): - if value: # Only show non-empty metadata - display_key = escape(key.replace("_", " ").title()) - html += f""" - - - - -""" - - html += """ - -
FieldValue
{display_key}{escape(str(value))}
-""" - - html += """ -
-

📊 Key Performance Indicators

-""" - - if scalar_kpis: - html += """ -

Scalar Metrics

- - - - - - - - - - - -""" - - for kpi in scalar_kpis: - direction_class = ( - "higher-better" if kpi["labels"]["higher_is_better"] else "lower-better" - ) - direction_text = ( - "↑ Higher is better" if kpi["labels"]["higher_is_better"] else "↓ Lower is better" - ) - - # Format the value using decorator format string - value = kpi["value"] - if isinstance(value, int | float): - try: - formatted_value = kpi["format"].format(value) - except (ValueError, KeyError): - formatted_value = str(value) - else: - formatted_value = str(value) - - html += f""" - - - - - - - -""" - - html += """ - -
KPIValueUnitDirectionDescription
{escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())}{escape(str(formatted_value))}{escape(str(kpi["unit"]))}{escape(direction_text)}{escape(str(kpi.get("help", "No description")))}
-""" - - if curve_kpis: - html += """ -

2D Metrics (Performance Curves)

- - - - - - - - - - - -""" - - for kpi in curve_kpis: - # Format 2D data points using decorator format strings - data_points = kpi["value"] - if data_points and len(data_points) > 0: - x_format = kpi.get("x_format", "{:.1f}") - y_format = kpi.get("y_format", "{:.1f}") - formatted_points = [] - for x, y in data_points[:5]: # Show first 5 points - try: - x_str = x_format.format(x) - y_str = y_format.format(y) - formatted_points.append(f"({x_str}, {y_str})") - except (ValueError, TypeError): - formatted_points.append(f"({x}, {y})") - points_text = ", ".join(formatted_points) - if len(data_points) > 5: - points_text += f" ... (+{len(data_points) - 5} more)" - else: - points_text = "No data" - - html += f""" - - - - - - - -""" - - html += """ - -
KPIData PointsX-AxisY-AxisDescription
{escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())}{escape(points_text)}{escape(str(kpi.get("x_unit", "unknown")))} - {escape(str(kpi.get("x_help", "")))}{escape(str(kpi.get("y_unit", kpi["unit"])))} - {escape(str(kpi.get("y_help", "")))}{escape(str(kpi.get("help", "No description")))}
-""" - - html += """ -
- -
- Generated by GuideLLM Caliper Plugin -
-
- - -""" - - return html - - -def _generate_multi_test_kpi_report( - records: list[Any], - output_dir: Path, - report_title: str = "GuideLLM KPI Report", - report_number: int | None = None, -) -> str: - """ - Generate a KPI report for multiple test records. - - Args: - records: List of unified result records (2 or more) - output_dir: Directory to write the HTML report - report_title: Title for the report - - Returns: - Path to the generated HTML file - """ - from projects.caliper.engine.kpi import get_kpi_functions, is_curve_kpi - from projects.guidellm.postprocess.guidellm.parsing import kpis as kpis_module - from projects.llm_d.postprocess.llm_d.parsing.kpis import GuideLLMKpiHandler - - kpi_functions = get_kpi_functions(kpis_module) - - # Process each record to extract test data - test_sections = [] - - for i, record in enumerate(records, 1): - logger.info(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") - - # Extract test condition labels - test_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(record) - base_labels = {**record.distinguishing_labels} - all_labels = {**base_labels, **test_labels} - - # Compute KPIs for this record - kpi_data = [] - - for kpi_id, kpi_func in kpi_functions.items(): - try: - value = kpi_func(record) - except (TypeError, ValueError, KeyError): - if is_curve_kpi(kpi_func): - value = [] # Empty list for failed curve KPIs - else: - value = None # None for missing/failed scalar KPIs - - # Create structured KPI record using core dataclass - kpi_record = KpiRecord( - schema_version="1", - kpi_id=kpi_id, - value=value if value is not None else 0, # KpiRecord requires non-null value - unit=kpi_func._kpi_unit, - run_id=record.test_base_path, - timestamp="", # Not critical for display - labels={"higher_is_better": kpi_func._kpi_higher_is_better}, - metadata={ - "help": kpi_func._kpi_help, - "format": getattr(kpi_func, "_kpi_format", "{:.2f}"), - "is_curve": is_curve_kpi(kpi_func), - }, - ) - - # Add 2D-specific metadata if applicable - if is_curve_kpi(kpi_func): - kpi_record.x_unit = kpi_func._kpi_x_unit - kpi_record.x_help = kpi_func._kpi_x_help - kpi_record.y_unit = getattr(kpi_func, "_kpi_y_unit", None) or kpi_func._kpi_unit - kpi_record.y_help = getattr(kpi_func, "_kpi_y_help", None) or kpi_func._kpi_help - # Add formatting info to metadata - kpi_record.metadata["x_format"] = getattr(kpi_func, "_kpi_x_format", "{:.1f}") - kpi_record.metadata["y_format"] = getattr(kpi_func, "_kpi_y_format", "{:.1f}") - - # Skip null values for display (can't create KpiRecord with None value) - if value is not None: - kpi_data.append(kpi_record.to_dict()) - - # Extract metadata and test info - metadata = GuideLLMKpiHandler.extract_metadata(record) - test_info = _extract_test_info(record) - - test_sections.append( - { - "record": record, - "test_info": test_info, - "labels": all_labels, - "kpi_data": kpi_data, - "metadata": metadata, - "section_title": f"Test {i}: {record.test_base_path}", - } - ) - - # Generate multi-test HTML - html_content = _generate_multi_test_html( - report_title=report_title, - test_sections=test_sections, - ) - - # Write to file - filename = create_report_filename("kpi_summary", report_number, report_title, "html") - output_file = output_dir / filename - output_file.write_text(html_content, encoding="utf-8") - - return str(output_file) - - -def _generate_multi_test_html( - report_title: str, - test_sections: list[dict], -) -> str: - """Generate HTML for multi-test KPI report.""" - - html_parts = [] - - # HTML header - html_parts.append(f""" - - - - - {escape(report_title)} - - - -
-

📊 {escape(report_title)}

- -
-

📋 Test Run Summary

-

Total Tests: {len(test_sections)} test configurations processed

-

Test Paths:

-
    """) - - # Add test paths overview - for i, section in enumerate(test_sections, 1): - html_parts.append( - f"
  • Test {i}: {escape(section['test_info']['test_path'])}
    • " - ) - - html_parts.append("""
-
-
-""") - - # Add section for each test - for section in test_sections: - html_parts.append(_generate_test_section_html(section)) - - # Footer - html_parts.append(""" -
-
- Generated by GuideLLM Caliper Plugin - Multi-Test KPI Report -
-
- -""") - - return "".join(html_parts) - - -def _generate_test_section_html(section: dict) -> str: - """Generate HTML for a single test section.""" - test_info = section["test_info"] - labels = section["labels"] - kpi_data = section["kpi_data"] - section_title = section["section_title"] - - # Sort KPIs - scalar_kpis = [kpi for kpi in kpi_data if not kpi.get("is_curve", False)] - curve_kpis = [kpi for kpi in kpi_data if kpi.get("is_curve", False)] - scalar_kpis.sort(key=lambda k: k["kpi_id"]) - curve_kpis.sort(key=lambda k: k["kpi_id"]) - - html_parts = [] - - html_parts.append(f""" -
-
-
-

🔬 {escape(section_title)}

-
- -
- Test Run: {escape(str(test_info["test_path"]))}
- Duration: {escape(str(test_info["duration"]))} | - Requests: {escape(str(test_info["successful_requests"]))}/{escape(str(test_info["total_requests"]))} successful -
- -

🏷️ Test Conditions

- - - - - - - - """) - - # Add test condition labels - for label, value in sorted(labels.items()): - if label != "higher_is_better": # Skip internal labels - html_parts.append(f""" - - - - """) - - html_parts.append(""" - -
LabelValue
{escape(label.replace("_", " ").title())}{escape(str(value))}
""") - - # Add KPIs - html_parts.append(""" -
-

📊 Key Performance Indicators

""") - - if scalar_kpis: - html_parts.append(""" -

Scalar Metrics

- - - - - - - - - - - """) - - for kpi in scalar_kpis: - direction_class = ( - "higher-better" if kpi["labels"]["higher_is_better"] else "lower-better" - ) - direction_text = ( - "↑ Higher is better" if kpi["labels"]["higher_is_better"] else "↓ Lower is better" - ) - - # Format the value - value = kpi["value"] - if isinstance(value, int | float): - try: - formatted_value = kpi["format"].format(value) - except (ValueError, KeyError): - formatted_value = str(value) - else: - formatted_value = str(value) - - html_parts.append(f""" - - - - - - - """) - - html_parts.append(""" - -
KPIValueUnitDirectionDescription
{escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())}{escape(str(formatted_value))}{escape(str(kpi["unit"]))}{escape(direction_text)}{escape(str(kpi.get("help", "No description")))}
""") - - if curve_kpis: - html_parts.append(""" -

2D Metrics (Performance Curves)

- - - - - - - - - - - """) - - for kpi in curve_kpis: - # Format 2D data points - data_points = kpi["value"] - if data_points and len(data_points) > 0: - x_format = kpi.get("x_format", "{:.1f}") - y_format = kpi.get("y_format", "{:.1f}") - formatted_points = [] - for x, y in data_points[:5]: # Show first 5 points - try: - x_str = x_format.format(x) - y_str = y_format.format(y) - formatted_points.append(f"({x_str}, {y_str})") - except (ValueError, TypeError): - formatted_points.append(f"({x}, {y})") - points_text = ", ".join(formatted_points) - if len(data_points) > 5: - points_text += f" ... (+{len(data_points) - 5} more)" - else: - points_text = "No data" - - html_parts.append(f""" - - - - - - - """) - - html_parts.append(""" - -
KPIData PointsX-AxisY-AxisDescription
{escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())}{escape(points_text)}{escape(str(kpi.get("x_unit", "unknown")))} - {escape(str(kpi.get("x_help", "")))}{escape(str(kpi.get("y_unit", kpi["unit"])))} - {escape(str(kpi.get("y_help", "")))}{escape(str(kpi.get("help", "No description")))}
""") - - html_parts.append(""" -
-
-
""") - - return "".join(html_parts) diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index 308bfe97d..799ab03c1 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -32,7 +32,6 @@ def _ensure_loaded(self) -> dict[str, dict[str, Any]]: if self._registry is not None: return self._registry - from .plotting.kpi_report import generate_kpi_report from .plotting.performance_analysis import ( generate_comprehensive_performance_report, generate_deployment_profile_report, @@ -48,15 +47,6 @@ def _ensure_loaded(self) -> dict[str, dict[str, Any]]: }, "description": "comprehensive performance analysis report (recommended)", }, - "report_kpi_summary": { - "function": generate_kpi_report, - "type": "report", - "kwargs": { - "report_number": 1, - "report_title": "GuideLLM KPI Summary", - }, - "description": "KPI summary with test conditions and metrics", - }, "report_deployment_profile": { "function": generate_deployment_profile_report, "type": "report", diff --git a/projects/guidellm/postprocess/guidellm/visualize-groups.yaml b/projects/guidellm/postprocess/guidellm/visualize-groups.yaml index 8b28c2f6a..a762472c7 100644 --- a/projects/guidellm/postprocess/guidellm/visualize-groups.yaml +++ b/projects/guidellm/postprocess/guidellm/visualize-groups.yaml @@ -4,29 +4,20 @@ # # Report system: # Report 00: GuideLLM Performance Analysis (comprehensive embedded report with all charts) -# Report 01: GuideLLM KPI Summary (test conditions and key performance indicators) -# Report 02: GuideLLM Deployment Profile Analysis (dedicated plots for each deployment profile) +# Report 01: GuideLLM Deployment Profile Analysis (dedicated plots for each deployment profile) groups: - # Comprehensive analysis - all reports including performance analysis and KPI summary (recommended) comprehensive: - - report_kpi_summary - report_performance_analysis release_regression: - report_deployment_profile - - report_kpi_summary # Performance overview - main performance report only overview: - report_performance_analysis - # KPI focus - test conditions and metrics summary only - kpi: - - report_kpi_summary - # All reports - complete set of available reports all: - report_performance_analysis - - report_kpi_summary - report_deployment_profile From c9cad721c02d7953201bc7044a508c4e50617868 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 22:39:48 +0200 Subject: [PATCH 0779/1072] [caliper] templates: kpi_listing: add blocks to allow iterating --- projects/caliper/templates/kpi_listing.html | 65 +++++++++++---------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/projects/caliper/templates/kpi_listing.html b/projects/caliper/templates/kpi_listing.html index 5e165c53d..e1a777ed9 100644 --- a/projects/caliper/templates/kpi_listing.html +++ b/projects/caliper/templates/kpi_listing.html @@ -219,41 +219,42 @@

No Data Available

{% for test in tests %} {% for kpi in test.kpis %} {% if kpi.kpi_id == kpi_id and kpi.is_curve and kpi.values_list and kpi.values_list|length > 0 %} - // Data for {{ test.run_id }} - let rawData = {{ kpi.values_list|tojson|safe }}; + (function() { // IIFE for {{ test.run_id }} + const rawData = {{ kpi.values_list|tojson|safe }}; - if (!firstKpi) { - firstKpi = { - name: '{{ kpi.name or kpi_id }}', - unit: '{{ kpi.unit or "" }}', - x_unit: '{{ kpi.x_unit or "" }}', - help: '{{ kpi.help or "" }}', - x_help: '{{ kpi.x_help or "" }}', - y_help: '{{ kpi.y_help or "" }}' - }; - } + if (!firstKpi) { + firstKpi = { + name: '{{ kpi.name or kpi_id }}', + unit: '{{ kpi.unit or "" }}', + x_unit: '{{ kpi.x_unit or "" }}', + help: '{{ kpi.help or "" }}', + x_help: '{{ kpi.x_help or "" }}', + y_help: '{{ kpi.y_help or "" }}' + }; + } - if (Array.isArray(rawData) && rawData.length > 0) { - let x_values = rawData.map(point => point[0]); - let y_values = rawData.map(point => point[1]); + if (Array.isArray(rawData) && rawData.length > 0) { + const x_values = rawData.map(point => point[0]); + const y_values = rawData.map(point => point[1]); - traces.push({ - x: x_values, - y: y_values, - type: 'scatter', - mode: 'lines+markers', - name: '{{ test.run_id }}', - line: { - color: colors[colorIndex % colors.length], - width: 2 - }, - marker: { - size: 6, - color: colors[colorIndex % colors.length] - } - }); - colorIndex++; - } + traces.push({ + x: x_values, + y: y_values, + type: 'scatter', + mode: 'lines+markers', + name: '{{ test.run_id }}', + line: { + color: colors[colorIndex % colors.length], + width: 2 + }, + marker: { + size: 6, + color: colors[colorIndex % colors.length] + } + }); + colorIndex++; + } + })(); {% endif %} {% endfor %} {% endfor %} From c5c2962069fb885d6f6aaddd46961c222f320ebd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 8 Sep 2026 23:01:49 +0200 Subject: [PATCH 0780/1072] [caliper] orchestration: caliper_invocation: expose the html_file in the postprocess_status --- projects/caliper/orchestration/caliper_invocation.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py index 4ef3cdca4..d7fdd9472 100644 --- a/projects/caliper/orchestration/caliper_invocation.py +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -420,6 +420,17 @@ def run_analyse_kpis( # Convert relative paths back to absolute for proper path handling if status_data.get("output_file"): status_data["output_file"] = str(Path(status_data["output_file"]).resolve()) + if status_data.get("html_file"): + # Make html_file relative to output_dir for MLflow compatibility + html_path = Path(status_data["html_file"]) + if html_path.is_absolute(): + try: + status_data["html_file"] = str(html_path.relative_to(output_dir)) + except ValueError: + # If can't make relative, use resolved absolute path as fallback + status_data["html_file"] = str(html_path.resolve()) + else: + status_data["html_file"] = str(html_path) # Ensure required fields are present with defaults status_data.setdefault("completed_at", time.time()) From 9e3a748bbcb64ee040c3a8e2b939aebafbcd2f20 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 08:44:35 +0200 Subject: [PATCH 0781/1072] [caliper] engine: visualize: validate the args before the parsing step --- projects/caliper/engine/visualize.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index c0c71b6ae..64b2449a0 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -71,6 +71,21 @@ def run_visualize( cache_path: Path | None, verbose_parsing: bool = False, ) -> list[str]: + # Validate visualization configuration BEFORE parsing to avoid wasted work + cfg = resolve_visualize_config(base_dir, visualize_config_path) + ids = resolve_report_ids( + reports_csv=reports_csv, + report_group=report_group, + config=cfg, + ) + + # Check if any report IDs were provided (early validation) + if not ids: + raise ValueError( + "No report IDs provided. Use --reports with comma-separated report IDs " + "or --report-group with a valid visualize config file." + ) + # Parse include and exclude filters for directory-level filtering include_filters = None exclude_filters = None @@ -92,19 +107,6 @@ def run_visualize( verbose_parsing=verbose_parsing, show_parameter_matrix=True, ) - cfg = resolve_visualize_config(base_dir, visualize_config_path) - ids = resolve_report_ids( - reports_csv=reports_csv, - report_group=report_group, - config=cfg, - ) - - # Check if any report IDs were provided - if not ids: - raise ValueError( - "No report IDs provided. Use --reports with comma-separated report IDs " - "or --report-group with a valid visualize config file." - ) output_dir.mkdir(parents=True, exist_ok=True) viz = plugin.visualize From a39484b03845fbb2d608045faba409a6396379c8 Mon Sep 17 00:00:00 2001 From: Harika Pothina Date: Thu, 10 Sep 2026 15:16:00 -0400 Subject: [PATCH 0782/1072] [rhaiis] Rename benchmark-heterogeneous to benchmark-heavy-heterogeneous --- projects/rhaiis/orchestration/presets.d/benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/presets.d/benchmarks.yaml b/projects/rhaiis/orchestration/presets.d/benchmarks.yaml index 2676051b2..b73b3d306 100644 --- a/projects/rhaiis/orchestration/presets.d/benchmarks.yaml +++ b/projects/rhaiis/orchestration/presets.d/benchmarks.yaml @@ -26,7 +26,7 @@ benchmark-multi-turn-prefix-off: - prefix-cache-off tests.rhaiis.workload_key: profile6 -benchmark-heterogeneous: +benchmark-heavy-heterogeneous: extends: - benchmark tests.rhaiis.workload_key: profile7 From e83a7b88703f5652370482f8f544266206924d38 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:37:18 +0200 Subject: [PATCH 0783/1072] [core] ci_entrypoint: prepare_ci: reduce the log level of git warnings --- projects/core/ci_entrypoint/prepare_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 96c38fc7c..893a78d03 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -326,10 +326,10 @@ def ci_banner(project: str, operation: str, args: list[str]): base_sha = os.environ.get("PULL_BASE_SHA", "main") if base_sha == "main": - logger.warning("PULL_BASE_SHA not set. Showing the last commits from main.") + logger.info("PULL_BASE_SHA not set. Showing the last commits from main.") pull_sha = os.environ.get("PULL_PULL_SHA", "") if not pull_sha: - logger.warning("PULL_PULL_SHA not set. Showing the last commits from main.") + logger.info("PULL_PULL_SHA not set. Showing the last commits from main.") logger.info(f"Git command will be: git show --quiet --oneline {base_sha}..{pull_sha}") From 251132833ef26cbc4d0cc6f6f100a02d5a09fa70 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:36:49 +0200 Subject: [PATCH 0784/1072] [core] library: ci: accept the exit reason --- projects/core/library/ci.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index d58ec9e04..3abea996d 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -169,12 +169,19 @@ def safe_ci_entrypoint(command_func): @functools.wraps(command_func) def wrapper(*args, **kwargs): exit_code = 0 + reason = None try: result = command_func(*args, **kwargs) - exit_code = result if result is not None else 0 + if result is None: + exit_code = 0 + elif isinstance(result, tuple) and len(result) == 2: + exit_code, reason = result + else: + exit_code = result except Exception as e: handle_ci_exception(e) exit_code = 1 + reason = str(e) # Save exit status to YAML file try: @@ -183,11 +190,18 @@ def wrapper(*args, **kwargs): exit_status_file = metadata_dir / "exit_status.yaml" exit_status_data = {"return_code": exit_code} + if reason is not None: + exit_status_data["reason"] = reason with open(exit_status_file, "w", encoding="utf-8") as f: yaml.dump(exit_status_data, f, default_flow_style=False) - logger.info(f"Exit status saved: {exit_status_file} (return_code: {exit_code})") + if reason: + logger.info( + f"Exit status saved: {exit_status_file} (return_code: {exit_code}, reason: {reason})" + ) + else: + logger.info(f"Exit status saved: {exit_status_file} (return_code: {exit_code})") except Exception as save_error: logger.warning(f"Failed to save exit status: {save_error}") @@ -232,6 +246,7 @@ def add_notification_file( f.write(message) logger.info(f"Created notification file: {file_path}") + logger.info(f"{message}") return str(file_path) except Exception as e: From 8b5bea4507f801cdff5fb9dc913c29b1599ee53f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 27 Jul 2026 14:42:45 +0200 Subject: [PATCH 0785/1072] [core] library: export: improve the notification mechanism --- .../caliper/orchestration/notification.py | 112 ++++--- projects/core/library/export.py | 315 +++++++++++++++--- 2 files changed, 327 insertions(+), 100 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 03d916304..2b089d79c 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -24,62 +24,63 @@ def format_postprocess_status_notification( Returns: Formatted notification text to include in GitHub notification """ - if not status: - return "" + + if not status or not status.steps: + return [] lines = [] # Check overall status (keep unchanged regardless of abort status) status_emoji = "✅" if status.is_success() else "❌" - lines.append(f"**Post-processing Status** {status_emoji}") - - # Add steps information if available, sorted by completion time - if status.steps: - # Convert steps from list[dict] format to sorted list of (step_name, step_data) tuples - step_tuples = [] - for step_dict in status.steps: - for step_name, step_data in step_dict.items(): - step_tuples.append((step_name, step_data)) - - # Sort by completion timestamp, with fallback to step name for stable ordering - sorted_steps = sorted( - step_tuples, - key=lambda item: ( - item[1].get("completed_at", 0) or 0, # Use completed_at if available, else 0 - item[0], # fallback to step name for stable ordering - ), - ) - - for step_name, step_data in sorted_steps: - step_emoji = _get_step_emoji(step_data.get("status", "unknown")) - - # Create step name as link to log file if available - log_file = step_data.get("log_file") - if log_file and get_file_link: - try: - log_url = get_file_link(log_file) - step_name_display = f"[**{step_name}**]({log_url})" - except Exception: - # Fallback to plain text if link generation fails - step_name_display = f"**{step_name}**" - else: + base_directory = status.base_directory + if base_directory.name == "status_files": + base_directory = base_directory.parent + + lines.append(f"**Post-processing Status** {status_emoji} `{base_directory}`") + + # Convert steps from list[dict] format to sorted list of (step_name, step_data) tuples + step_tuples = [] + for step_dict in status.steps: + for step_name, step_data in step_dict.items(): + step_tuples.append((step_name, step_data)) + + # Sort by completion timestamp, with fallback to step name for stable ordering + sorted_steps = sorted( + step_tuples, + key=lambda item: ( + item[1].get("completed_at", 0) or 0, # Use completed_at if available, else 0 + item[0], # fallback to step name for stable ordering + ), + ) + + for step_name, step_data in sorted_steps: + step_emoji = _get_step_emoji(step_data.get("status", "unknown")) + + # Create step name as link to log file if available + log_file = step_data.get("log_file") + if log_file and get_file_link: + try: + log_url = get_file_link(log_file) + step_name_display = f"[**{step_name}**]({log_url})" + except Exception: + # Fallback to plain text if link generation fails step_name_display = f"**{step_name}**" + else: + step_name_display = f"**{step_name}**" - # Format step with message if available - lines.append( - f"- {step_emoji} {step_name_display}: `{step_data.get('status', 'unknown')}`" - ) - message = step_data.get("message") - if message: - lines.append(f" * `{message}`") + # Format step with message if available + lines.append(f"- {step_emoji} {step_name_display}: `{step_data.get('status', 'unknown')}`") + message = step_data.get("message") + if message: + lines.append(f" * `{message}`") - reason = step_data.get("reason") - if reason: - lines.append(f" * `{reason}`") + reason = step_data.get("reason") + if reason: + lines.append(f" * `{reason}`") - # Use object-oriented step formatter to handle step-specific details - step_details = _format_step_details_with_formatters(step_name, step_data, get_file_link) - lines.extend(step_details) + # Use object-oriented step formatter to handle step-specific details + step_details = _format_step_details_with_formatters(step_name, step_data, get_file_link) + lines.extend(step_details) return "\n".join(lines) if lines else "" @@ -125,15 +126,16 @@ def _create_file_link(file_path: str, emoji: str, get_file_link: callable | None def _format_artifacts_to_kpis_step(step_data: dict, get_file_link: callable | None) -> list[str]: """Format artifacts_to_kpis step details.""" lines = [] - output_file = step_data.get("output_file") - if output_file: - lines.append(_create_file_link(output_file, "📄", get_file_link)) # Include HTML file if available html_file = step_data.get("html_file") if html_file: lines.append(_create_file_link(html_file, "🌐", get_file_link)) + output_file = step_data.get("output_file") + if output_file: + lines.append(_create_file_link(output_file, "📄", get_file_link)) + return lines @@ -229,6 +231,11 @@ def _format_analyse_kpis_step(step_data: dict, get_file_link: callable | None) - """Format analyse_kpis step details.""" lines = [] + # Include HTML file if available + html_file = step_data.get("html_file") + if html_file: + lines.append(_create_file_link(html_file, "🌐", get_file_link)) + # Show analysis output file output_file = step_data.get("output_file") if output_file: @@ -260,11 +267,6 @@ def _format_analyse_kpis_step(step_data: dict, get_file_link: callable | None) - if baseline_files_count is not None: lines.append(f" - 📈 Baseline files analyzed: `{baseline_files_count}`") - # Include HTML file if available - html_file = step_data.get("html_file") - if html_file: - lines.append(_create_file_link(html_file, "🌐", get_file_link)) - return lines diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 5bae8f607..bf3d234cb 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -98,10 +98,13 @@ def _update_fjob_export_status(status: dict): os.environ["KUBECONFIG"] = original_kubeconfig -def send_notification(status: dict[str, Any], notification_provider=None) -> bool: +def send_notification( + artifact_dir: Path | None, status: dict[str, Any], notification_provider=None +) -> bool: """Send job completion notifications based on caliper export status. Args: + artifact_dir: Directory to browse to find the artifacts status: Caliper export status object containing backend results and metadata notification_provider: Optional per-project SlackNotificationProvider instance @@ -124,19 +127,21 @@ def send_notification(status: dict[str, Any], notification_provider=None) -> boo logger.info(f"Sending notification: {project} {operation} {finish_reason}{duration_str}") # Build enhanced notification with fournos job info and artifact links - notification_status = _build_enhanced_notification(project, finish_reason, duration_str, status) + notification_status = _build_enhanced_notification( + artifact_dir, project, finish_reason, duration_str, status + ) # Write notification to file for GitHub pickup try: if env.ARTIFACT_DIR: notification_file = Path(env.ARTIFACT_DIR) / "NOTIFICATION-github.md" with open(notification_file, "w", encoding="utf-8") as f: - f.write(notification_status) + f.write(notification_status + "\n") logger.info(f"Wrote export notification file {notification_file}") else: logger.warning("ARTIFACT_DIR not available, skipping notification file") except Exception as e: - logger.warning(f"Failed to write notification file: {e}") + logger.exception(f"Failed to write notification file: {e}") # Actually send notification through GitHub API try: @@ -318,6 +323,10 @@ def _extract_artifact_links(status: dict[str, Any]) -> tuple[list[str], str | No def _create_mlflow_url(mlflow_run_url: str, step_dir_name: str) -> str | None: """Create MLflow URL for step logs.""" + + if not mlflow_run_url: + return f"BASE_URL_MISSING/{step_dir_name}" + if "/artifacts" not in mlflow_run_url: logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") return None @@ -512,17 +521,17 @@ def _process_notification_files(step_dir: Path, step_log_links: list[str]) -> No continue -def _process_step_logs(mlflow_run_url: str) -> list[str]: +def _process_step_logs(artifact_dir: Path, mlflow_run_url: str) -> list[str]: """Process step logs from parent directory.""" + if not mlflow_run_url: - logging.warning("mlflow_run_url not set, skipping step log browsing") - return [] + logging.warning("mlflow_run_url not set. Will generate dummy links.") step_log_links = [] - parent_dir = Path(env.BASE_ARTIFACT_DIR).parent + current_step_name = Path(env.BASE_ARTIFACT_DIR).name - for step_dir in sorted(parent_dir.iterdir()): + for step_dir in sorted(artifact_dir.iterdir()): if not step_dir.is_dir(): continue if step_dir.name.startswith("."): @@ -639,15 +648,14 @@ def _check_postprocess_warnings(step_dir: Path) -> StepStatus: return status -def _get_overall_status_from_steps() -> str: +def _get_overall_status_from_steps(artifact_dir) -> str: """Check all step exit statuses and return overall status emoji.""" try: - parent_dir = Path(env.BASE_ARTIFACT_DIR).parent current_step_name = Path(env.BASE_ARTIFACT_DIR).name step_statuses = [] - for step_dir in sorted(parent_dir.iterdir()): + for step_dir in sorted(artifact_dir.iterdir()): if not step_dir.is_dir(): continue if step_dir.name.startswith("."): @@ -684,7 +692,11 @@ def _get_overall_status_from_steps() -> str: def _build_enhanced_notification( - project: str, finish_reason: FinishReason, duration_str: str, status: dict[str, Any] + artifact_dir: Path, + project: str, + finish_reason: FinishReason, + duration_str: str, + status: dict[str, Any], ) -> str: """Build enhanced notification with fournos job config and artifact links.""" fjob_project, fjob_args_str = _get_project_and_args(project) @@ -695,7 +707,7 @@ def _build_enhanced_notification( status_emoji = "🛑" # Abort status overrides everything else: # Check all step statuses for overall status emoji (takes priority over finish_reason) - status_emoji = _get_overall_status_from_steps() + status_emoji = _get_overall_status_from_steps(artifact_dir) base_status = f"**{status_emoji} Execution of `{fjob_project}` {fjob_args_str} {status_emoji}**" @@ -714,7 +726,7 @@ def _build_enhanced_notification( try: artifact_links, mlflow_run_url = _extract_artifact_links(status) - step_log_links = _process_step_logs(mlflow_run_url) + step_log_links = _process_step_logs(artifact_dir, mlflow_run_url) postprocess_status_links = _process_postprocess_status(mlflow_run_url) if artifact_links: @@ -724,31 +736,199 @@ def _build_enhanced_notification( else: notification_parts.append("**Artifact Links:** No direct links available") + # Filter step_log_links to separate regular test logs from export-artifacts + regular_test_links = [] + export_artifacts_links = [] + if step_log_links: + for link in step_log_links: + # Check if this is an export-artifacts link (contains "Export" in the name) + if "Export" in link or "export" in link.lower(): + export_artifacts_links.append(link) + else: + regular_test_links.append(link) + + # Add regular test logs first + if regular_test_links: notification_parts.append("") + notification_parts.append("---") notification_parts.append("**Test Logs**") - notification_parts.extend(step_log_links) + notification_parts.extend(regular_test_links) # Add distinct test and post-processing status right under Test Logs - test_status_section = _build_test_status_section(status) + test_status_section = _build_test_status_section(status, artifact_dir, mlflow_run_url) if test_status_section: notification_parts.append("") notification_parts.extend(test_status_section) + # Add export-artifacts links at the end + if export_artifacts_links: + notification_parts.append("") + notification_parts.extend(export_artifacts_links) + if postprocess_status_links: notification_parts.append("") notification_parts.extend(postprocess_status_links) except Exception as e: - logger.warning(f"Failed to extract artifact links: {e}") + logger.exception(f"Failed to extract artifact links: {e}") notification_parts.append("**Artifact Links:** Error extracting links") raise return "\n".join(notification_parts) -def _build_test_status_section(status: dict[str, Any]) -> list[str]: - """Build distinct test and post-processing status section.""" +def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: + """Extract test execution information from __test_labels__.yaml files. + + Args: + artifact_dir: Directory to search for __test_labels__.yaml files + mlflow_run_url: Optional MLflow run URL for creating links + + Returns: + List of formatted strings with test information (directory, labels, success, message) + """ + from pathlib import Path + + import yaml + + test_info_lines = [] + + # Search for __test_labels__.yaml files recursively + test_labels_files = list(artifact_dir.glob("**/__test_labels__.yaml")) + + if not test_labels_files: + return ["No test labels found"] + + for test_labels_file in test_labels_files: + try: + with open(test_labels_file, encoding="utf-8") as f: + test_data = yaml.safe_load(f) or {} + + # Extract directory relative to artifact_dir - use just the immediate directory name + relative_dir = test_labels_file.parent.relative_to(artifact_dir) + dir_name = relative_dir.name if relative_dir != Path(".") else "root" + + # Extract completion info + completion = test_data.get("completion", {}) + success = completion.get("success") + message = completion.get("message") + + if message: + message = f" `{message}`" + # Format status + if success: + status_emoji = "✅" + elif success is False: + status_emoji = "❌" + else: + status_emoji = "❓" + + # Create link to __test_labels__.yaml file if MLflow URL is available + if mlflow_run_url: + try: + # Get the step directory name (relative to parent) + step_dir_name = str(relative_dir) + test_labels_url = _create_mlflow_file_url_for_step( + mlflow_run_url, step_dir_name, "__test_labels__.yaml" + ) + dir_link = f"[**{dir_name}**]({test_labels_url})" + except Exception as e: + logger.warning(f"Failed to create MLflow link for {test_labels_file}: {e}") + dir_link = f"**{dir_name}**" + else: + dir_link = f"**{dir_name}**" + + test_info_lines.append(f"* {status_emoji} {dir_link}:{message}") + + except Exception as e: + test_info_lines.append(f"**{test_labels_file.name}**: Error reading file - {e}") + + return test_info_lines + + +def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: + """Extract post-processing status information from postprocess_status.yaml files. + + Returns: + List of formatted strings with postprocess step status (success only, no details) + """ + from pathlib import Path + + import yaml + + postprocess_info_lines = [] + + # Search for postprocess_status.yaml files recursively + postprocess_files = list(artifact_dir.glob("**/postprocess_status.yaml")) + + if not postprocess_files: + return ["No postprocess status found"] + + for postprocess_file in postprocess_files: + try: + with open(postprocess_file, encoding="utf-8") as f: + postprocess_data = yaml.safe_load(f) or {} + + # Extract directory relative to artifact_dir + relative_dir = postprocess_file.parent.relative_to(artifact_dir) + dir_name = str(relative_dir) if relative_dir != Path(".") else "root" + + # Extract overall status + overall_success = postprocess_data.get("success", False) + final_status = postprocess_data.get("final_status", "unknown") + + # Extract individual step statuses + steps = postprocess_data.get("steps", []) + step_statuses = [] + + for step_dict in steps: + for step_name, step_data in step_dict.items(): + if isinstance(step_data, dict): + status = step_data.get("status", "unknown") + status_emoji = ( + "✅" if status == "success" else "❌" if status == "failed" else "⚪" + ) + step_statuses.append(f"{step_name}:{status_emoji}") + + # Format overall line + overall_emoji = "✅" if overall_success else "❌" + if step_statuses: + steps_str = " " + " ".join(step_statuses) + else: + steps_str = f" {final_status}" + + postprocess_info_lines.append(f"**{dir_name}**: {overall_emoji}{steps_str}") + + except Exception as e: + postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") + + return postprocess_info_lines + + +def _process_step_status(step_dir: Path, mlflow_run_url: str | None = None) -> dict[str, Any]: + """Process test labels and postprocess status for a single step directory.""" + step_info = {"step_name": step_dir.name, "test_labels_info": [], "postprocess_info": []} + + # Extract test labels for this specific step + try: + step_info["test_labels_info"] = _extract_test_labels_info(step_dir, mlflow_run_url) + except Exception as e: + logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") + + # Extract postprocess status for this specific step + try: + step_info["postprocess_info"] = _extract_postprocess_status_info(step_dir) + except Exception as e: + logger.warning(f"Failed to extract postprocess status for step {step_dir.name}: {e}") + + return step_info + + +def _build_test_status_section( + status: dict[str, Any], artifact_dir: Path, mlflow_run_url: str | None = None +) -> list[str]: + """Build comprehensive test status section with step-by-step execution and post-processing overviews.""" try: test_phase = status.get("test_phase", {}) if not test_phase: @@ -776,6 +956,40 @@ def _build_test_status_section(status: dict[str, Any]) -> list[str]: status_lines.append(f"**post-processing:** {post_processing_status}") + # Process each step directory for test execution and post-processing info + step_overview_lines = [] + postprocess_overview_lines = [] + + for step_dir in sorted(artifact_dir.iterdir()): + if not step_dir.is_dir() or step_dir.name.startswith("."): + continue + + step_status = _process_step_status(step_dir, mlflow_run_url) + + # Add test execution info for this step + if step_status["test_labels_info"]: + step_overview_lines.extend( + [f"* {info}" for info in step_status["test_labels_info"]] + ) + + # Add postprocess info for this step + if step_status["postprocess_info"]: + postprocess_overview_lines.extend( + [f"* {info}" for info in step_status["postprocess_info"]] + ) + + # Add Test Execution Overview if we have any step info + if step_overview_lines: + status_lines.append("") + status_lines.append("**Test Execution Overview**") + status_lines.extend(step_overview_lines) + + # Add Post-Processing Overview if we have any step info + if postprocess_overview_lines: + status_lines.append("") + status_lines.append("**Post-Processing Overview**") + status_lines.extend(postprocess_overview_lines) + return status_lines except Exception as e: @@ -839,14 +1053,7 @@ def _should_skip_notification(project: str, operation: str, finish_reason: Finis return False -def run_caliper_orchestration_export(*, artifact_directory: Path | None): - """Set optional ``caliper.export.from`` and run orchestration export.""" - - if artifact_directory is None and "ARTIFACT_BASE_DIR" in os.environ: - artifact_directory = os.environ["ARTIFACT_BASE_DIR"] - - if artifact_directory is not None: - config.project.set_config("caliper.export.from", str(artifact_directory)) +def run_caliper_orchestration_export(*, artifact_dir: Path): # Use FJOB_NAME as fallback for mlflow run_name if not configured run_name = config.project.get_config( @@ -901,15 +1108,15 @@ def run_caliper_orchestration_export(*, artifact_directory: Path | None): @click.command("export-artifacts") @click.option( - "--artifact-directory", - "artifact_directory", + "--artifact-dir", + "artifact_dir", type=click.Path(path_type=Path, exists=False, file_okay=True, dir_okay=True), default=None, help="If set, overrides caliper.export.from (artifact root directory).", ) @click.pass_context @ci_lib.safe_ci_entrypoint -def caliper_export_entrypoint(_ctx, artifact_directory: Path | None): +def caliper_export_entrypoint(_ctx, artifact_dir: Path | None): """Export the file artifacts.""" notification_provider = getattr(getattr(_ctx, "obj", None), "notification_provider", None) @@ -918,8 +1125,35 @@ def caliper_export_entrypoint(_ctx, artifact_directory: Path | None): export_failed = False notification_failed = False + # Determine artifact directory with proper precedence and FOURNOS_CI handling + if not artifact_dir: + # First try the config field + artifact_dir = config.project.get_config( + "caliper.export.from", None, print=False, warn=False + ) + + if not artifact_dir and env.ARTIFACT_DIR: + artifact_dir = env.ARTIFACT_DIR + logger.info(f"Using ARTIFACT_DIR from environment: {artifact_dir}") + # Apply FOURNOS_CI logic only when using ARTIFACT_DIR + if os.environ.get("FOURNOS_CI") == "true": + artifact_dir = Path(artifact_dir).parent + logger.info(f"FOURNOS_CI=true: using parent directory: {artifact_dir}") + + if not artifact_dir: + logger.error( + "No artifact directory found. Please set --artifact-dir parameter, " + "caliper.export.from config, ARTIFACT_DIR, or ARTIFACT_BASE_DIR environment variable." + ) + return 1 + + logging.info(f"Building caliper notification from {artifact_dir}") + + # Set the config so other functions can access it + config.project.set_config("caliper.export.from", str(artifact_dir)) + try: - status = run_caliper_orchestration_export(artifact_directory=artifact_directory) + status = run_caliper_orchestration_export(artifact_dir=artifact_dir) logger.info("Export status:\n" + yaml.dump(status, indent=4)) # Update fjob status with export results @@ -936,7 +1170,7 @@ def caliper_export_entrypoint(_ctx, artifact_directory: Path | None): if status: try: notification_success = send_notification( - status, notification_provider=notification_provider + artifact_dir, status, notification_provider=notification_provider ) if not notification_success: logger.error("Notification sending failed") @@ -945,7 +1179,7 @@ def caliper_export_entrypoint(_ctx, artifact_directory: Path | None): logger.exception(f"Failed to send notifications: {e}") notification_failed = True - _update_final_artifacts(status) + _update_final_artifacts(artifact_dir, status) # Return proper exit code if export_failed or notification_failed: @@ -953,7 +1187,7 @@ def caliper_export_entrypoint(_ctx, artifact_directory: Path | None): return 0 -def _update_final_artifacts(export_status: dict[str, Any] | None) -> None: +def _update_final_artifacts(artifact_dir, export_status: dict[str, Any] | None) -> None: """Update the final artifacts (run.log, notifications) to MLflow after all post-export work is done.""" if not export_status: logger.warning("No export status received, cannot update the final artifacts") @@ -973,16 +1207,7 @@ def _update_final_artifacts(export_status: dict[str, Any] | None) -> None: if not run_id: return - artifact_from = config.project.get_config( - "caliper.export.from", None, print=False, warn=False - ) - if not artifact_from: - logger.warning( - "Export status don't have the caliper.export.from field, cannot update the final artifacts" - ) - return - - artifact_root = Path(artifact_from) + artifact_root = Path(artifact_dir) artifact_path = str(env.ARTIFACT_DIR.relative_to(artifact_root)) tracking_uri = mlflow_meta.get("tracking_uri") From 4289fecbb00e3a926d99f1d93e333f10d39d2c83 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 17:04:40 +0200 Subject: [PATCH 0786/1072] [core] library: export: add the dry-run flag --- projects/core/library/export.py | 174 ++++++++++++++++++-------------- 1 file changed, 101 insertions(+), 73 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index bf3d234cb..742e80b08 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -99,7 +99,10 @@ def _update_fjob_export_status(status: dict): def send_notification( - artifact_dir: Path | None, status: dict[str, Any], notification_provider=None + artifact_dir: Path | None, + status: dict[str, Any], + notification_provider=None, + dry_run: bool = False, ) -> bool: """Send job completion notifications based on caliper export status. @@ -107,37 +110,41 @@ def send_notification( artifact_dir: Directory to browse to find the artifacts status: Caliper export status object containing backend results and metadata notification_provider: Optional per-project SlackNotificationProvider instance + dry_run: If True, only build and log notification content without sending Returns: bool: True if notifications were sent successfully, False otherwise """ # Extract notification parameters from status object - project = _extract_project_from_status(status) - operation = _extract_operation_from_status(status) + + project = config.project.get_config("project.name") finish_reason = _extract_finish_reason_from_status(status) duration_str = _extract_duration_from_status(status) - # Apply minimal filtering logic - if _should_skip_notification(project, operation, finish_reason): - logger.info(f"Skipping notification for {project} {operation}") - return True # Skipped is considered success - - # Send actual notifications - notification_success = True - logger.info(f"Sending notification: {project} {operation} {finish_reason}{duration_str}") - # Build enhanced notification with fournos job info and artifact links notification_status = _build_enhanced_notification( artifact_dir, project, finish_reason, duration_str, status ) - # Write notification to file for GitHub pickup + # Send actual notifications + notification_success = True + if dry_run: + logger.info(f"DRY RUN: Would send notification: {project} {finish_reason}{duration_str}") + logger.info(f"DRY RUN: Notification content:\n{notification_status}") + return True # Dry run is considered success + else: + logger.info(f"Sending notification: {project} {finish_reason}{duration_str}") + + # Write notification to file for GitHub pickup (always generate, even in dry-run) try: if env.ARTIFACT_DIR: notification_file = Path(env.ARTIFACT_DIR) / "NOTIFICATION-github.md" with open(notification_file, "w", encoding="utf-8") as f: f.write(notification_status + "\n") - logger.info(f"Wrote export notification file {notification_file}") + if dry_run: + logger.info(f"DRY RUN: Generated notification file {notification_file}") + else: + logger.info(f"Wrote export notification file {notification_file}") else: logger.warning("ARTIFACT_DIR not available, skipping notification file") except Exception as e: @@ -150,8 +157,6 @@ def send_notification( # Get notification vault from configuration notification_vault = None try: - from projects.core.library import config - notification_config = config.project.get_config("caliper.export.notifications", {}) notification_vault = notification_config.get("vault") if notification_vault: @@ -163,7 +168,7 @@ def send_notification( message=notification_status, github=True, slack=False, - dry_run=False, + dry_run=dry_run, notification_vault=notification_vault, ) if success: @@ -177,27 +182,30 @@ def send_notification( # Per-project Slack notification via provider if notification_provider: - try: - from projects.core.notifications.provider import NotificationContext - - artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None - context = NotificationContext( - status=status, - finish_reason=str(finish_reason), - project_name=project or "unknown", - pr_number=os.environ.get("PULL_NUMBER"), - job_type=os.environ.get("JOB_TYPE"), - artifact_dir=artifact_dir, - ) - ok = notification_provider.notify(context) - if ok: - logger.info("Successfully sent per-project Slack notification") - else: - logger.warning("Per-project Slack notification failed") + if not dry_run: + try: + from projects.core.notifications.provider import NotificationContext + + artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None + context = NotificationContext( + status=status, + finish_reason=str(finish_reason), + project_name=project or "unknown", + pr_number=os.environ.get("PULL_NUMBER"), + job_type=os.environ.get("JOB_TYPE"), + artifact_dir=artifact_dir, + ) + ok = notification_provider.notify(context) + if ok: + logger.info("Successfully sent per-project Slack notification") + else: + logger.warning("Per-project Slack notification failed") + notification_success = False + except Exception as e: + logger.warning(f"Failed to send per-project Slack notification: {e}") notification_success = False - except Exception as e: - logger.warning(f"Failed to send per-project Slack notification: {e}") - notification_success = False + else: + logger.info("DRY RUN: Would send per-project Slack notification") return notification_success @@ -475,7 +483,15 @@ def _process_caliper_postprocess_status( # Calculate step subdirectory relative to BASE_ARTIFACT_DIR.parent # e.g., "/workspace/artifacts/000__replot/postprocess_output" relative to "/workspace/artifacts" = "000__replot/postprocess_output" - step_subdir = str(base_path.relative_to(env.BASE_ARTIFACT_DIR.parent)) + try: + step_subdir = str(base_path.relative_to(env.BASE_ARTIFACT_DIR.parent)) + except ValueError as e: + # Path resolution failed (common in dry-run or different directory contexts) + logger.warning( + f"Failed to resolve path {base_path} relative to {env.BASE_ARTIFACT_DIR.parent}: {e}" + ) + # Fallback: use the step directory name + step_subdir = step_dir.name else: # Fallback to step_dir.name for backward compatibility step_subdir = step_dir.name @@ -997,27 +1013,6 @@ def _build_test_status_section( return [] -def _extract_project_from_status(status: dict[str, Any]) -> str: - """Extract project name from status object or environment.""" - # Try to get project from environment variables - project = os.environ.get("PROJECT_NAME") - if project: - return project - - # Fallback to JOB_NAME parsing (common in CI environments) - job_name = os.environ.get("JOB_NAME", "") - if job_name and "-" in job_name: - # Extract project from job name pattern like "project-operation-variant" - return job_name.split("-")[0] - - return "unknown" - - -def _extract_operation_from_status(status: dict[str, Any]) -> str: - """Extract operation name from status object.""" - return "export-artifacts" - - def _extract_finish_reason_from_status(status: dict[str, Any]) -> FinishReason: """Extract finish reason from status object.""" # Check if any backend failed in the status @@ -1047,12 +1042,6 @@ def _extract_duration_from_status(status: dict[str, Any]) -> str: return "" -def _should_skip_notification(project: str, operation: str, finish_reason: FinishReason) -> bool: - """Apply minimal filtering logic to determine if notification should be skipped.""" - # Minimal filtering - no special cases for now - return False - - def run_caliper_orchestration_export(*, artifact_dir: Path): # Use FJOB_NAME as fallback for mlflow run_name if not configured @@ -1114,9 +1103,16 @@ def run_caliper_orchestration_export(*, artifact_dir: Path): default=None, help="If set, overrides caliper.export.from (artifact root directory).", ) +@click.option( + "--dry-run", + "dry_run", + is_flag=True, + default=False, + help="Show what would be exported and notified without actually performing operations.", +) @click.pass_context @ci_lib.safe_ci_entrypoint -def caliper_export_entrypoint(_ctx, artifact_dir: Path | None): +def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): """Export the file artifacts.""" notification_provider = getattr(getattr(_ctx, "obj", None), "notification_provider", None) @@ -1147,17 +1143,43 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None): ) return 1 - logging.info(f"Building caliper notification from {artifact_dir}") + if dry_run: + logging.info(f"DRY RUN: Building caliper notification from {artifact_dir}") + else: + logging.info(f"Building caliper notification from {artifact_dir}") # Set the config so other functions can access it config.project.set_config("caliper.export.from", str(artifact_dir)) try: - status = run_caliper_orchestration_export(artifact_dir=artifact_dir) - logger.info("Export status:\n" + yaml.dump(status, indent=4)) + if dry_run: + logger.info( + "DRY RUN: Skipping actual caliper export, creating mock status for notification" + ) + # Create a realistic mock status for notification testing + status = { + "success": True, + "final_status": "success", + "backends": {}, + "caliper_artifacts_export": { + "backends": { + "mlflow": { + "success": True, + "run_id": "dry-run-mock-id", + "experiment_url": "http://localhost:5000/#/experiments/123", + "run_url": "http://localhost:5000/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", + "tracking_uri": "http://localhost:5000", + } + } + }, + "duration": "15 minutes, 30 seconds", + } + else: + status = run_caliper_orchestration_export(artifact_dir=artifact_dir) + logger.info("Export status:\n" + yaml.dump(status, indent=4)) - # Update fjob status with export results - _update_fjob_export_status(status) + # Update fjob status with export results + _update_fjob_export_status(status) except Exception as e: logger.error(f"Export failed: {e}") @@ -1170,7 +1192,10 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None): if status: try: notification_success = send_notification( - artifact_dir, status, notification_provider=notification_provider + artifact_dir, + status, + notification_provider=notification_provider, + dry_run=dry_run, ) if not notification_success: logger.error("Notification sending failed") @@ -1179,7 +1204,10 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None): logger.exception(f"Failed to send notifications: {e}") notification_failed = True - _update_final_artifacts(artifact_dir, status) + if not dry_run: + _update_final_artifacts(artifact_dir, status) + else: + logger.info("DRY RUN: Skipping final artifacts update to MLflow") # Return proper exit code if export_failed or notification_failed: From f1e54993a8d1a9e9509ff1dbb7b72b172d6be8f6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 17:52:36 +0200 Subject: [PATCH 0787/1072] [core] library: export: improve the notification mechanism --- projects/core/library/export.py | 200 ++++++++++++-------------------- 1 file changed, 77 insertions(+), 123 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 742e80b08..46cefe99e 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -122,16 +122,15 @@ def send_notification( duration_str = _extract_duration_from_status(status) # Build enhanced notification with fournos job info and artifact links - notification_status = _build_enhanced_notification( + notification_status, notification_success = _build_enhanced_notification( artifact_dir, project, finish_reason, duration_str, status ) # Send actual notifications - notification_success = True if dry_run: logger.info(f"DRY RUN: Would send notification: {project} {finish_reason}{duration_str}") logger.info(f"DRY RUN: Notification content:\n{notification_status}") - return True # Dry run is considered success + return notification_success else: logger.info(f"Sending notification: {project} {finish_reason}{duration_str}") @@ -474,6 +473,7 @@ def _process_caliper_postprocess_status( if mlflow_run_url: # Use base_directory from status data for MLflow URL construction base_directory = result.base_directory + if base_directory: # Calculate path relative to BASE_ARTIFACT_DIR.parent # e.g., "/workspace/artifacts/000__replot/postprocess_output" -> "000__replot/postprocess_output" @@ -509,41 +509,39 @@ def get_file_link(file_path: str, step_subdir=step_subdir) -> str: raise -def _process_notification_files(step_dir: Path, step_log_links: list[str]) -> None: +def _process_notification_files(step_dir: Path) -> None: """Process notification files from step directory.""" notifications_dir = step_dir / CI_METADATA_DIRNAME / "notifications" if not (notifications_dir.exists() and notifications_dir.is_dir()): - return + return [] import re + notifications_from_files = [] for notification_file in sorted(notifications_dir.glob("*.txt")): - try: - with open(notification_file, encoding="utf-8") as f: - content = f.read().strip() + with open(notification_file, encoding="utf-8") as f: + content = f.read().strip() - if not content: - continue + if not content: + continue - subtitle = notification_file.stem.replace("__", " ").replace("_", " ").title() - subtitle = re.sub(r"^\d+\s+", "", subtitle) - step_log_links.append(f"##### {subtitle}") + subtitle = notification_file.stem.replace("__", " ").replace("_", " ").title() + subtitle = re.sub(r"^\d+\s+", "", subtitle) + notifications_from_files.append(f"##### {subtitle}") - for line in content.splitlines(): - step_log_links.append(f"> {line}") + for line in content.splitlines(): + notifications_from_files.append(f"> {line}") - except Exception as file_error: - logger.warning(f"Failed to read notification file {notification_file}: {file_error}") - continue + return notifications_from_files -def _process_step_logs(artifact_dir: Path, mlflow_run_url: str) -> list[str]: +def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: """Process step logs from parent directory.""" if not mlflow_run_url: logging.warning("mlflow_run_url not set. Will generate dummy links.") - step_log_links = [] + step_status = [] current_step_name = Path(env.BASE_ARTIFACT_DIR).name @@ -557,29 +555,39 @@ def _process_step_logs(artifact_dir: Path, mlflow_run_url: str) -> list[str]: if not run_log.exists(): continue - try: - mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_dir.name) - if not mlflow_log_url: - continue + mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_dir.name) + if not mlflow_log_url: + mlflow_log_url = "NO_URL" - step_name = step_dir.name.replace("__", " ").replace("_", " ").title() - duration_str = _read_step_duration(step_dir) - exit_status_emoji, exit_status = _read_step_exit_status(step_dir, current_step_name) + step_name = step_dir.name.replace("__", " ").replace("_", " ").title() + duration_str = _read_step_duration(step_dir) + exit_status_emoji, exit_status = _read_step_exit_status(step_dir, current_step_name) - if duration_str: - step_log_links.append( - f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}) `{duration_str}`" - ) - else: - step_log_links.append(f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url})") + step_status.append("") + if duration_str: + step_status.append( + f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}) `{duration_str}`" + ) + else: + step_status.append(f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url})") - _process_notification_files(step_dir, step_log_links) + step_status.extend(_process_notification_files(step_dir)) - except Exception as e: - logger.warning(f"Failed to create MLflow link for {run_log}: {e}") - continue + step_details = _process_step_details(step_dir, mlflow_run_url) + + # Add test execution info for this step + if step_details["test_labels_info"]: + # Add Test Execution Overview if we have any step info + step_status.append("") + step_status.append("**Test Execution Overview**") - return step_log_links + step_status.extend([f"* {info}" for info in step_details["test_labels_info"]]) + + # Add postprocess info for this step + if step_details["postprocess_info"]: + step_status.extend([f"* {info}" for info in step_details["postprocess_info"]]) + + return step_status def _process_postprocess_status(mlflow_run_url: str | None = None) -> list[str]: @@ -740,58 +748,46 @@ def _build_enhanced_notification( notification_parts.append("**Execution Engine Configuration**") notification_parts.append(execution_engine_config) + notification_success = True try: artifact_links, mlflow_run_url = _extract_artifact_links(status) - step_log_links = _process_step_logs(artifact_dir, mlflow_run_url) + step_status = _process_step_status(artifact_dir, mlflow_run_url) postprocess_status_links = _process_postprocess_status(mlflow_run_url) + test_status_section = _build_test_status_section(status, artifact_dir, mlflow_run_url) + + if test_status_section: + notification_parts.append("") + notification_parts.append("---") + notification_parts.extend(test_status_section) if artifact_links: notification_parts.append("") + notification_parts.append("---") notification_parts.append("**Artifact Links**") notification_parts.extend([f"* {link}" for link in artifact_links]) else: notification_parts.append("**Artifact Links:** No direct links available") - # Filter step_log_links to separate regular test logs from export-artifacts - regular_test_links = [] - export_artifacts_links = [] - - if step_log_links: - for link in step_log_links: - # Check if this is an export-artifacts link (contains "Export" in the name) - if "Export" in link or "export" in link.lower(): - export_artifacts_links.append(link) - else: - regular_test_links.append(link) - - # Add regular test logs first - if regular_test_links: + if step_status: notification_parts.append("") notification_parts.append("---") - notification_parts.append("**Test Logs**") - notification_parts.extend(regular_test_links) - - # Add distinct test and post-processing status right under Test Logs - test_status_section = _build_test_status_section(status, artifact_dir, mlflow_run_url) - if test_status_section: - notification_parts.append("") - notification_parts.extend(test_status_section) - - # Add export-artifacts links at the end - if export_artifacts_links: - notification_parts.append("") - notification_parts.extend(export_artifacts_links) + notification_parts.append("**Step details**") + for link in step_status: + notification_parts.append(link) if postprocess_status_links: notification_parts.append("") notification_parts.extend(postprocess_status_links) except Exception as e: - logger.exception(f"Failed to extract artifact links: {e}") + logger.exception(f"Failed to build the extended notifications: {e}") notification_parts.append("**Artifact Links:** Error extracting links") - raise + notification_success = False + + notification_parts.append("") + notification_parts.append("---") - return "\n".join(notification_parts) + return "\n".join(notification_parts), notification_success def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: @@ -814,7 +810,7 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N test_labels_files = list(artifact_dir.glob("**/__test_labels__.yaml")) if not test_labels_files: - return ["No test labels found"] + return [] for test_labels_file in test_labels_files: try: @@ -831,7 +827,8 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N message = completion.get("message") if message: - message = f" `{message}`" + message = f": `{message}`" + # Format status if success: status_emoji = "✅" @@ -855,7 +852,7 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N else: dir_link = f"**{dir_name}**" - test_info_lines.append(f"* {status_emoji} {dir_link}:{message}") + test_info_lines.append(f"* {status_emoji} {dir_link}{message}") except Exception as e: test_info_lines.append(f"**{test_labels_file.name}**: Error reading file - {e}") @@ -879,7 +876,7 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: postprocess_files = list(artifact_dir.glob("**/postprocess_status.yaml")) if not postprocess_files: - return ["No postprocess status found"] + return [] for postprocess_file in postprocess_files: try: @@ -922,7 +919,7 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: return postprocess_info_lines -def _process_step_status(step_dir: Path, mlflow_run_url: str | None = None) -> dict[str, Any]: +def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> dict[str, Any]: """Process test labels and postprocess status for a single step directory.""" step_info = {"step_name": step_dir.name, "test_labels_info": [], "postprocess_info": []} @@ -953,59 +950,12 @@ def _build_test_status_section( test_status = test_phase.get("phase", "UNKNOWN") test_message = test_phase.get("message", "") - # Determine post-processing status based on final status and test outcome - final_status = status.get("final_status", "unknown") - if test_status == "FAILED": - post_processing_status = "skipped" # Don't run post-processing if test failed - elif final_status == "success": - post_processing_status = "success" - elif "failed" in final_status.lower(): - post_processing_status = "failed" - else: - post_processing_status = "unknown" - - status_lines = [f"**test:** {test_status}"] + status_lines = [f"**Test status**: {test_status}"] if test_message: # Format message with blockquote-style prefix status_lines.append(f"> {test_message}") - status_lines.append(f"**post-processing:** {post_processing_status}") - - # Process each step directory for test execution and post-processing info - step_overview_lines = [] - postprocess_overview_lines = [] - - for step_dir in sorted(artifact_dir.iterdir()): - if not step_dir.is_dir() or step_dir.name.startswith("."): - continue - - step_status = _process_step_status(step_dir, mlflow_run_url) - - # Add test execution info for this step - if step_status["test_labels_info"]: - step_overview_lines.extend( - [f"* {info}" for info in step_status["test_labels_info"]] - ) - - # Add postprocess info for this step - if step_status["postprocess_info"]: - postprocess_overview_lines.extend( - [f"* {info}" for info in step_status["postprocess_info"]] - ) - - # Add Test Execution Overview if we have any step info - if step_overview_lines: - status_lines.append("") - status_lines.append("**Test Execution Overview**") - status_lines.extend(step_overview_lines) - - # Add Post-Processing Overview if we have any step info - if postprocess_overview_lines: - status_lines.append("") - status_lines.append("**Post-Processing Overview**") - status_lines.extend(postprocess_overview_lines) - return status_lines except Exception as e: @@ -1173,6 +1123,10 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): } }, "duration": "15 minutes, 30 seconds", + "test_phase": { + "phase": "FAILED", + "message": "Test execution completed with failures", + }, } else: status = run_caliper_orchestration_export(artifact_dir=artifact_dir) @@ -1209,9 +1163,9 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): else: logger.info("DRY RUN: Skipping final artifacts update to MLflow") - # Return proper exit code if export_failed or notification_failed: return 1 + return 0 From 35f9bcace1d2e704ce217acb41cea3d036118a09 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:46:58 +0200 Subject: [PATCH 0788/1072] [core] library: export: move notification code to a dedicated file --- projects/core/library/export.py | 843 +-------------- projects/core/library/export_notifications.py | 982 ++++++++++++++++++ projects/core/library/step_status.py | 15 + 3 files changed, 1005 insertions(+), 835 deletions(-) create mode 100644 projects/core/library/export_notifications.py create mode 100644 projects/core/library/step_status.py diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 46cefe99e..44061096c 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -16,21 +16,16 @@ import click import yaml -from projects.caliper.orchestration.export import run_from_orchestration_config -from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME +from projects.caliper.orchestration.export import ( + run_from_orchestration_config, +) from projects.core.library import ci as ci_lib from projects.core.library import config, env, run - - -class StepStatus(StrEnum): - """Status of a step execution.""" - - SUCCESS = "success" - FAILURE = "failure" - ONGOING = "ongoing" - UNKNOWN = "unknown" - WARNING = "warning" - +from projects.core.library.export_notifications import ( + _check_job_shutdown_status, + _create_mlflow_file_url_for_step, + send_notification, +) logger = logging.getLogger(__name__) @@ -98,345 +93,6 @@ def _update_fjob_export_status(status: dict): os.environ["KUBECONFIG"] = original_kubeconfig -def send_notification( - artifact_dir: Path | None, - status: dict[str, Any], - notification_provider=None, - dry_run: bool = False, -) -> bool: - """Send job completion notifications based on caliper export status. - - Args: - artifact_dir: Directory to browse to find the artifacts - status: Caliper export status object containing backend results and metadata - notification_provider: Optional per-project SlackNotificationProvider instance - dry_run: If True, only build and log notification content without sending - - Returns: - bool: True if notifications were sent successfully, False otherwise - """ - # Extract notification parameters from status object - - project = config.project.get_config("project.name") - finish_reason = _extract_finish_reason_from_status(status) - duration_str = _extract_duration_from_status(status) - - # Build enhanced notification with fournos job info and artifact links - notification_status, notification_success = _build_enhanced_notification( - artifact_dir, project, finish_reason, duration_str, status - ) - - # Send actual notifications - if dry_run: - logger.info(f"DRY RUN: Would send notification: {project} {finish_reason}{duration_str}") - logger.info(f"DRY RUN: Notification content:\n{notification_status}") - return notification_success - else: - logger.info(f"Sending notification: {project} {finish_reason}{duration_str}") - - # Write notification to file for GitHub pickup (always generate, even in dry-run) - try: - if env.ARTIFACT_DIR: - notification_file = Path(env.ARTIFACT_DIR) / "NOTIFICATION-github.md" - with open(notification_file, "w", encoding="utf-8") as f: - f.write(notification_status + "\n") - if dry_run: - logger.info(f"DRY RUN: Generated notification file {notification_file}") - else: - logger.info(f"Wrote export notification file {notification_file}") - else: - logger.warning("ARTIFACT_DIR not available, skipping notification file") - except Exception as e: - logger.exception(f"Failed to write notification file: {e}") - - # Actually send notification through GitHub API - try: - from projects.core.notifications.send import send_notification as send_github_notification - - # Get notification vault from configuration - notification_vault = None - try: - notification_config = config.project.get_config("caliper.export.notifications", {}) - notification_vault = notification_config.get("vault") - if notification_vault: - logger.info(f"Using notification vault from config: {notification_vault}") - except Exception as e: - logger.warning(f"Failed to get notification vault from config: {e}") - - success = send_github_notification( - message=notification_status, - github=True, - slack=False, - dry_run=dry_run, - notification_vault=notification_vault, - ) - if success: - logger.info("Successfully sent GitHub notification") - else: - logger.error("GitHub notification sending failed") - notification_success = False - except Exception as e: - logger.error(f"Failed to send GitHub notification: {e}") - notification_success = False - - # Per-project Slack notification via provider - if notification_provider: - if not dry_run: - try: - from projects.core.notifications.provider import NotificationContext - - artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None - context = NotificationContext( - status=status, - finish_reason=str(finish_reason), - project_name=project or "unknown", - pr_number=os.environ.get("PULL_NUMBER"), - job_type=os.environ.get("JOB_TYPE"), - artifact_dir=artifact_dir, - ) - ok = notification_provider.notify(context) - if ok: - logger.info("Successfully sent per-project Slack notification") - else: - logger.warning("Per-project Slack notification failed") - notification_success = False - except Exception as e: - logger.warning(f"Failed to send per-project Slack notification: {e}") - notification_success = False - else: - logger.info("DRY RUN: Would send per-project Slack notification") - - return notification_success - - -def _get_project_and_args(project: str) -> tuple[str, str]: - """Extract project name and args from fournos job or config.""" - fjob_project = project - fjob_args_str = "" - - try: - metadata_dir = ci_lib.get_ci_metadata_dir() - fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" - if not fournos_fjob_path.exists(): - return fjob_project, fjob_args_str - - with open(fournos_fjob_path, encoding="utf-8") as f: - fjob_data = yaml.safe_load(f) - - display_name = fjob_data.get("spec", {}).get("displayName", "") - if not display_name: - return fjob_project, fjob_args_str - - parts = display_name.split() - if not parts: - return fjob_project, fjob_args_str - - fjob_project = parts[0] - fjob_args_str = " ".join(parts[1:]) if len(parts) > 1 else "" - except Exception as e: - logger.warning(f"Failed to read fournos job for project/args: {e}") - - if fjob_args_str: - return fjob_project, fjob_args_str - - try: - from projects.core.library import config - - job_args = config.project.get_config("ci_job.args", [], warn=False) - fjob_args_str = " ".join(job_args) if job_args else "" - except Exception as e: - logger.warning(f"Failed to get args from config: {e}") - - return fjob_project, fjob_args_str - - -def _get_execution_engine_config() -> str | None: - """Read and format execution engine configuration.""" - try: - metadata_dir = ci_lib.get_ci_metadata_dir() - fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" - if not fournos_fjob_path.exists(): - return None - - with open(fournos_fjob_path, encoding="utf-8") as f: - fjob_data = yaml.safe_load(f) - - execution_engine = fjob_data.get("spec", {}).get("executionEngine", {}) - if not execution_engine: - return None - - engine_yaml = yaml.dump(execution_engine, default_flow_style=False, sort_keys=True) - return f"```yaml\n{engine_yaml.strip()}\n```" - except Exception as e: - logger.warning(f"Failed to read fournos job config: {e}") - return None - - -def _check_job_shutdown_status() -> dict[str, Any] | None: - """Check if the job has been aborted via spec.shutdown field.""" - try: - metadata_dir = ci_lib.get_ci_metadata_dir() - fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" - if not fournos_fjob_path.exists(): - return None - - with open(fournos_fjob_path, encoding="utf-8") as f: - fjob_data = yaml.safe_load(f) - - shutdown_value = fjob_data.get("spec", {}).get("shutdown") - if shutdown_value: - return { - "shutdown_detected": True, - "shutdown_value": shutdown_value, - "is_aborted": shutdown_value.lower() == "stop", - } - - return {"shutdown_detected": False, "shutdown_value": None, "is_aborted": False} - except Exception as e: - logger.warning(f"Failed to check job shutdown status: {e}") - return None - - -def _extract_artifact_links(status: dict[str, Any]) -> tuple[list[str], str | None]: - """Extract artifact links and MLflow URL from status.""" - artifact_links = [] - mlflow_run_url = None - - caliper_export = status.get("caliper_artifacts_export", {}) - backends = caliper_export.get("backends", {}) - - for backend_name, backend_result in backends.items(): - if not isinstance(backend_result, dict): - continue - - if backend_result.get("experiment_url"): - artifact_links.append( - f"[{backend_name} Experiment]({backend_result['experiment_url']})" - ) - - if backend_result.get("run_url"): - mlflow_run_url = backend_result["run_url"] - artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") - elif backend_result.get("artifact_url"): - artifact_links.append(f"[{backend_name} Artifacts]({backend_result['artifact_url']})") - elif backend_result.get("dashboard_url"): - artifact_links.append(f"[{backend_name} Dashboard]({backend_result['dashboard_url']})") - - if status.get("artifact_url"): - artifact_links.append(f"[Artifacts]({status['artifact_url']})") - - return artifact_links, mlflow_run_url - - -def _create_mlflow_url(mlflow_run_url: str, step_dir_name: str) -> str | None: - """Create MLflow URL for step logs.""" - - if not mlflow_run_url: - return f"BASE_URL_MISSING/{step_dir_name}" - - if "/artifacts" not in mlflow_run_url: - logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") - return None - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, params = hash_fragment.split("/artifacts", 1) - workspace_param = params if "?workspace=" in params else "" - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/run.log{workspace_param}" - else: - base_url, params = mlflow_run_url.split("/artifacts", 1) - workspace_param = params if "?workspace=" in params else "" - return f"{base_url}/artifacts/{step_dir_name}/run.log{workspace_param}" - - -def _create_mlflow_step_url(mlflow_run_url: str, step_dir_name: str) -> str | None: - """Create MLflow URL for step directory (for file access).""" - if "/artifacts" not in mlflow_run_url: - logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") - return None - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}{workspace_param}" - else: - base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_url}/artifacts/{step_dir_name}{workspace_param}" - - -def _create_mlflow_file_url_for_step( - mlflow_run_url: str, step_dir_name: str, file_path: str -) -> str: - """Create MLflow URL for a specific file within a step directory. - - Args: - mlflow_run_url: Base MLflow run URL - step_dir_name: Name of the step directory - file_path: Relative path to file from step directory - - Returns: - Full MLflow URL to the file - - Raises: - ValueError: If URL format is unexpected - """ - if "/artifacts" not in mlflow_run_url: - raise ValueError(f"Unexpected MLflow URL format: {mlflow_run_url}") - - # Clean file path - file_clean = file_path.lstrip("/") - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" - else: - base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_url}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" - - -def _read_step_duration(step_dir: Path) -> str: - """Read step duration from timing file.""" - timing_file = step_dir / CI_METADATA_DIRNAME / "test_duration.yaml" - if not timing_file.exists(): - return "" - - try: - with open(timing_file, encoding="utf-8") as f: - timing_data = yaml.safe_load(f) - - formatted_duration = timing_data.get("duration", {}).get("formatted") - return formatted_duration or "" - except Exception as timing_error: - logger.warning(f"Failed to read timing file {timing_file}: {timing_error}") - return "" - - def _process_caliper_postprocess_status( step_dir: Path, step_log_links: list[str], mlflow_run_url: str | None = None ) -> None: @@ -509,489 +165,6 @@ def get_file_link(file_path: str, step_subdir=step_subdir) -> str: raise -def _process_notification_files(step_dir: Path) -> None: - """Process notification files from step directory.""" - notifications_dir = step_dir / CI_METADATA_DIRNAME / "notifications" - if not (notifications_dir.exists() and notifications_dir.is_dir()): - return [] - - import re - - notifications_from_files = [] - for notification_file in sorted(notifications_dir.glob("*.txt")): - with open(notification_file, encoding="utf-8") as f: - content = f.read().strip() - - if not content: - continue - - subtitle = notification_file.stem.replace("__", " ").replace("_", " ").title() - subtitle = re.sub(r"^\d+\s+", "", subtitle) - notifications_from_files.append(f"##### {subtitle}") - - for line in content.splitlines(): - notifications_from_files.append(f"> {line}") - - return notifications_from_files - - -def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: - """Process step logs from parent directory.""" - - if not mlflow_run_url: - logging.warning("mlflow_run_url not set. Will generate dummy links.") - - step_status = [] - - current_step_name = Path(env.BASE_ARTIFACT_DIR).name - - for step_dir in sorted(artifact_dir.iterdir()): - if not step_dir.is_dir(): - continue - if step_dir.name.startswith("."): - continue - - run_log = step_dir / "run.log" - if not run_log.exists(): - continue - - mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_dir.name) - if not mlflow_log_url: - mlflow_log_url = "NO_URL" - - step_name = step_dir.name.replace("__", " ").replace("_", " ").title() - duration_str = _read_step_duration(step_dir) - exit_status_emoji, exit_status = _read_step_exit_status(step_dir, current_step_name) - - step_status.append("") - if duration_str: - step_status.append( - f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}) `{duration_str}`" - ) - else: - step_status.append(f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url})") - - step_status.extend(_process_notification_files(step_dir)) - - step_details = _process_step_details(step_dir, mlflow_run_url) - - # Add test execution info for this step - if step_details["test_labels_info"]: - # Add Test Execution Overview if we have any step info - step_status.append("") - step_status.append("**Test Execution Overview**") - - step_status.extend([f"* {info}" for info in step_details["test_labels_info"]]) - - # Add postprocess info for this step - if step_details["postprocess_info"]: - step_status.extend([f"* {info}" for info in step_details["postprocess_info"]]) - - return step_status - - -def _process_postprocess_status(mlflow_run_url: str | None = None) -> list[str]: - """Process post-processing status from all step directories.""" - if not mlflow_run_url: - return [] - - postprocess_links = [] - parent_dir = Path(env.BASE_ARTIFACT_DIR).parent - - for step_dir in sorted(parent_dir.iterdir()): - if not step_dir.is_dir(): - continue - if step_dir.name.startswith("."): - continue - - try: - _process_caliper_postprocess_status(step_dir, postprocess_links, mlflow_run_url) - except Exception as e: - logger.error(f"Failed to process postprocess status for {step_dir.name}: {e}") - raise - - return postprocess_links - - -def _read_step_exit_status( - step_dir: Path, current_step_name: str | None = None -) -> tuple[str, StepStatus]: - """Read exit status from step directory and return emoji and status enum.""" - try: - exit_status_file = step_dir / CI_METADATA_DIRNAME / "exit_status.yaml" - if not exit_status_file.exists(): - # Check if this is the current ongoing step - if current_step_name and step_dir.name == current_step_name: - return "🔄", StepStatus.ONGOING # Ongoing step - return "❓", StepStatus.UNKNOWN # Unknown status if file doesn't exist - - with open(exit_status_file, encoding="utf-8") as f: - exit_data = yaml.safe_load(f) - - return_code = exit_data.get("return_code") - if return_code is None or return_code == 0: - return "✅", StepStatus.SUCCESS - else: - return "❌", StepStatus.FAILURE - except Exception as e: - logger.warning(f"Failed to read exit status from {step_dir}: {e}") - # Check if this is the current ongoing step even on error - if current_step_name and step_dir.name == current_step_name: - return "🔄", StepStatus.ONGOING # Ongoing step - return "❓", StepStatus.UNKNOWN # Unknown status on error - - -def _check_postprocess_warnings(step_dir: Path) -> StepStatus: - """Check for warning status in postprocess status file.""" - - status = StepStatus.SUCCESS # No postprocess warning/error, assume no warnings - for status_file in step_dir.glob("**/postprocess_status.yaml"): - try: - with open(status_file, encoding="utf-8") as f: - status_data = yaml.safe_load(f) - except Exception as e: - logging.error(f"Failed to read {status_file} as yaml: {e}") - status = StepStatus.WARNING - continue - - if not status_data: - continue - - # Check top-level success field for warning value - success_value = status_data.get("success") - if success_value == "warning": - logging.warning( - f"Post-process warning detected in {status_file}, setting the WARNING flag" - ) - status = StepStatus.WARNING - - if success_value in ("failure", "error"): - logging.error(f"Post-process {success_value} detected, raising the FAILURE flag") - return StepStatus.FAILURE - - return status - - -def _get_overall_status_from_steps(artifact_dir) -> str: - """Check all step exit statuses and return overall status emoji.""" - try: - current_step_name = Path(env.BASE_ARTIFACT_DIR).name - - step_statuses = [] - - for step_dir in sorted(artifact_dir.iterdir()): - if not step_dir.is_dir(): - continue - if step_dir.name.startswith("."): - continue - - # Only check directories that have run.log (actual steps) - run_log = step_dir / "run.log" - if not run_log.exists(): - continue - - _emoji, status = _read_step_exit_status(step_dir, current_step_name) - - step_statuses.append(status) - - # Check for postprocess warnings in this step (always check, regardless of exit status) - postprocess_status = _check_postprocess_warnings(step_dir) - step_statuses.append(postprocess_status) - - # Priority: failure > ongoing > warning > unknown > success - if StepStatus.FAILURE in step_statuses: - return "🔴" # Any failure = red - elif StepStatus.WARNING in step_statuses: - return "🟠" # Warning = orange - elif StepStatus.UNKNOWN in step_statuses: - return "🟠" # Unknown = orange - elif StepStatus.ONGOING in step_statuses: - return "🟢" # Ongoing --> success - else: - return "🟢" # All successful = green - - except Exception as e: - logger.exception(f"Failed to check step statuses: {e}") - return "🔴" # Error checking = red - - -def _build_enhanced_notification( - artifact_dir: Path, - project: str, - finish_reason: FinishReason, - duration_str: str, - status: dict[str, Any], -) -> str: - """Build enhanced notification with fournos job config and artifact links.""" - fjob_project, fjob_args_str = _get_project_and_args(project) - - # Check for job shutdown first (takes highest priority) - shutdown_status = _check_job_shutdown_status() - if shutdown_status and shutdown_status.get("is_aborted"): - status_emoji = "🛑" # Abort status overrides everything - else: - # Check all step statuses for overall status emoji (takes priority over finish_reason) - status_emoji = _get_overall_status_from_steps(artifact_dir) - - base_status = f"**{status_emoji} Execution of `{fjob_project}` {fjob_args_str} {status_emoji}**" - - notification_parts = [base_status, "---"] - - # Add job abort message right below overall status if applicable - if shutdown_status and shutdown_status.get("is_aborted"): - shutdown_value = shutdown_status.get("shutdown_value", "Stop") - notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") - notification_parts.append("") - - execution_engine_config = _get_execution_engine_config() - if execution_engine_config: - notification_parts.append("**Execution Engine Configuration**") - notification_parts.append(execution_engine_config) - - notification_success = True - try: - artifact_links, mlflow_run_url = _extract_artifact_links(status) - step_status = _process_step_status(artifact_dir, mlflow_run_url) - postprocess_status_links = _process_postprocess_status(mlflow_run_url) - test_status_section = _build_test_status_section(status, artifact_dir, mlflow_run_url) - - if test_status_section: - notification_parts.append("") - notification_parts.append("---") - notification_parts.extend(test_status_section) - - if artifact_links: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Artifact Links**") - notification_parts.extend([f"* {link}" for link in artifact_links]) - else: - notification_parts.append("**Artifact Links:** No direct links available") - - if step_status: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Step details**") - for link in step_status: - notification_parts.append(link) - - if postprocess_status_links: - notification_parts.append("") - notification_parts.extend(postprocess_status_links) - - except Exception as e: - logger.exception(f"Failed to build the extended notifications: {e}") - notification_parts.append("**Artifact Links:** Error extracting links") - notification_success = False - - notification_parts.append("") - notification_parts.append("---") - - return "\n".join(notification_parts), notification_success - - -def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: - """Extract test execution information from __test_labels__.yaml files. - - Args: - artifact_dir: Directory to search for __test_labels__.yaml files - mlflow_run_url: Optional MLflow run URL for creating links - - Returns: - List of formatted strings with test information (directory, labels, success, message) - """ - from pathlib import Path - - import yaml - - test_info_lines = [] - - # Search for __test_labels__.yaml files recursively - test_labels_files = list(artifact_dir.glob("**/__test_labels__.yaml")) - - if not test_labels_files: - return [] - - for test_labels_file in test_labels_files: - try: - with open(test_labels_file, encoding="utf-8") as f: - test_data = yaml.safe_load(f) or {} - - # Extract directory relative to artifact_dir - use just the immediate directory name - relative_dir = test_labels_file.parent.relative_to(artifact_dir) - dir_name = relative_dir.name if relative_dir != Path(".") else "root" - - # Extract completion info - completion = test_data.get("completion", {}) - success = completion.get("success") - message = completion.get("message") - - if message: - message = f": `{message}`" - - # Format status - if success: - status_emoji = "✅" - elif success is False: - status_emoji = "❌" - else: - status_emoji = "❓" - - # Create link to __test_labels__.yaml file if MLflow URL is available - if mlflow_run_url: - try: - # Get the step directory name (relative to parent) - step_dir_name = str(relative_dir) - test_labels_url = _create_mlflow_file_url_for_step( - mlflow_run_url, step_dir_name, "__test_labels__.yaml" - ) - dir_link = f"[**{dir_name}**]({test_labels_url})" - except Exception as e: - logger.warning(f"Failed to create MLflow link for {test_labels_file}: {e}") - dir_link = f"**{dir_name}**" - else: - dir_link = f"**{dir_name}**" - - test_info_lines.append(f"* {status_emoji} {dir_link}{message}") - - except Exception as e: - test_info_lines.append(f"**{test_labels_file.name}**: Error reading file - {e}") - - return test_info_lines - - -def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: - """Extract post-processing status information from postprocess_status.yaml files. - - Returns: - List of formatted strings with postprocess step status (success only, no details) - """ - from pathlib import Path - - import yaml - - postprocess_info_lines = [] - - # Search for postprocess_status.yaml files recursively - postprocess_files = list(artifact_dir.glob("**/postprocess_status.yaml")) - - if not postprocess_files: - return [] - - for postprocess_file in postprocess_files: - try: - with open(postprocess_file, encoding="utf-8") as f: - postprocess_data = yaml.safe_load(f) or {} - - # Extract directory relative to artifact_dir - relative_dir = postprocess_file.parent.relative_to(artifact_dir) - dir_name = str(relative_dir) if relative_dir != Path(".") else "root" - - # Extract overall status - overall_success = postprocess_data.get("success", False) - final_status = postprocess_data.get("final_status", "unknown") - - # Extract individual step statuses - steps = postprocess_data.get("steps", []) - step_statuses = [] - - for step_dict in steps: - for step_name, step_data in step_dict.items(): - if isinstance(step_data, dict): - status = step_data.get("status", "unknown") - status_emoji = ( - "✅" if status == "success" else "❌" if status == "failed" else "⚪" - ) - step_statuses.append(f"{step_name}:{status_emoji}") - - # Format overall line - overall_emoji = "✅" if overall_success else "❌" - if step_statuses: - steps_str = " " + " ".join(step_statuses) - else: - steps_str = f" {final_status}" - - postprocess_info_lines.append(f"**{dir_name}**: {overall_emoji}{steps_str}") - - except Exception as e: - postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") - - return postprocess_info_lines - - -def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> dict[str, Any]: - """Process test labels and postprocess status for a single step directory.""" - step_info = {"step_name": step_dir.name, "test_labels_info": [], "postprocess_info": []} - - # Extract test labels for this specific step - try: - step_info["test_labels_info"] = _extract_test_labels_info(step_dir, mlflow_run_url) - except Exception as e: - logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") - - # Extract postprocess status for this specific step - try: - step_info["postprocess_info"] = _extract_postprocess_status_info(step_dir) - except Exception as e: - logger.warning(f"Failed to extract postprocess status for step {step_dir.name}: {e}") - - return step_info - - -def _build_test_status_section( - status: dict[str, Any], artifact_dir: Path, mlflow_run_url: str | None = None -) -> list[str]: - """Build comprehensive test status section with step-by-step execution and post-processing overviews.""" - try: - test_phase = status.get("test_phase", {}) - if not test_phase: - return [] - - test_status = test_phase.get("phase", "UNKNOWN") - test_message = test_phase.get("message", "") - - status_lines = [f"**Test status**: {test_status}"] - - if test_message: - # Format message with blockquote-style prefix - status_lines.append(f"> {test_message}") - - return status_lines - - except Exception as e: - logger.warning(f"Failed to build test status section: {e}") - return [] - - -def _extract_finish_reason_from_status(status: dict[str, Any]) -> FinishReason: - """Extract finish reason from status object.""" - # Check if any backend failed in the status - if not status: - return FinishReason.ERROR - - # Look for backend results - backends = status.get("backends", {}) - for backend_name, backend_result in backends.items(): - # Check both explicit success flag and status field - if backend_result.get("success") is False or backend_result.get("status") not in ( - None, - "success", - ): - logger.info(f"Backend {backend_name} failed, marking as error") - return FinishReason.ERROR - - return FinishReason.SUCCESS - - -def _extract_duration_from_status(status: dict[str, Any]) -> str: - """Extract duration from status object.""" - # Look for duration in status - duration = status.get("duration") - if duration: - return f" after {duration}" - return "" - - def run_caliper_orchestration_export(*, artifact_dir: Path): # Use FJOB_NAME as fallback for mlflow run_name if not configured diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py new file mode 100644 index 000000000..ed033b268 --- /dev/null +++ b/projects/core/library/export_notifications.py @@ -0,0 +1,982 @@ +""" +Notification handling for Caliper export operations. + +This module provides notification functionality for export completion, +including GitHub notifications and Slack notifications via project providers. +""" + +import logging +import os +import re +from pathlib import Path +from typing import Any + +import yaml +from projects.caliper.orchestration.censoring import censor_text + +from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME +from projects.core.library import ci as ci_lib +from projects.core.library import config, env +from projects.core.library.step_status import StepStatus + +logger = logging.getLogger(__name__) + + +def _censor_notification_text(text: str, verbose: bool = False) -> str: + """ + Censor sensitive content in notification text using caliper orchestration. + + Args: + text: The notification text to censor + verbose: Enable verbose logging + + Returns: + Censored notification text with sensitive content replaced + """ + return censor_text(text, verbose=verbose) + + +def send_notification( + artifact_dir: Path | None, + status: dict[str, Any], + notification_provider=None, + dry_run: bool = False, +) -> bool: + """Send job completion notifications based on caliper export status. + + Args: + artifact_dir: Directory to browse to find the artifacts + status: Caliper export status object containing backend results and metadata + notification_provider: Optional per-project SlackNotificationProvider instance + dry_run: If True, only build and log notification content without sending + + Returns: + bool: True if notifications were sent successfully, False otherwise + """ + # Extract notification parameters from status object + project = config.project.get_config("project.name") + finish_reason = _extract_finish_reason_from_status(status) + + # Build enhanced notification with fournos job info and artifact links + notification_status, notification_success = _build_enhanced_notification( + artifact_dir, project, finish_reason, status + ) + + # Apply censoring to notification content before sending + notification_status = _censor_notification_text(notification_status, verbose=dry_run) + + # Send actual notifications + if dry_run: + logger.info("DRY RUN: Would send notification") + logger.info(f"DRY RUN: Notification content:\n{notification_status}") + else: + logger.info("Sending notification ...") + + # Write notification to file for GitHub pickup (always generate, even in dry-run) + try: + if env.ARTIFACT_DIR: + notification_file = Path(env.ARTIFACT_DIR) / "NOTIFICATION-github.md" + with open(notification_file, "w", encoding="utf-8") as f: + f.write(notification_status + "\n") + if dry_run: + logger.info(f"DRY RUN: Generated notification file {notification_file}") + else: + logger.info(f"Wrote export notification file {notification_file}") + else: + logger.warning("ARTIFACT_DIR not available, skipping notification file") + except Exception as e: + logger.exception(f"Failed to write notification file: {e}") + + # Actually send notification through GitHub API + try: + from projects.core.notifications.send import send_notification as send_github_notification + + # Get notification vault from configuration + notification_vault = None + try: + notification_config = config.project.get_config("caliper.export.notifications", {}) + notification_vault = notification_config.get("vault") + if notification_vault: + logger.info(f"Using notification vault from config: {notification_vault}") + except Exception as e: + logger.warning(f"Failed to get notification vault from config: {e}") + + success = send_github_notification( + message=notification_status, + github=True, + slack=False, + dry_run=dry_run, + notification_vault=notification_vault, + ) + if success: + logger.info("Successfully sent GitHub notification") + else: + logger.error("GitHub notification sending failed") + notification_success = False + except Exception as e: + logger.error(f"Failed to send GitHub notification: {e}") + notification_success = False + + # Per-project Slack notification via provider + if notification_provider: + if not dry_run: + try: + from projects.core.notifications.provider import NotificationContext + + artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None + context = NotificationContext( + status=status, + finish_reason=str(finish_reason), + project_name=project or "unknown", + pr_number=os.environ.get("PULL_NUMBER"), + job_type=os.environ.get("JOB_TYPE"), + artifact_dir=artifact_dir, + ) + ok = notification_provider.notify(context) + if ok: + logger.info("Successfully sent per-project Slack notification") + else: + logger.warning("Per-project Slack notification failed") + notification_success = False + except Exception as e: + logger.warning(f"Failed to send per-project Slack notification: {e}") + notification_success = False + else: + logger.info("DRY RUN: Would send per-project Slack notification") + + return notification_success + + +def _get_project_and_args(project: str) -> tuple[str, str]: + """Extract project name and args from fournos job or config.""" + fjob_project = project + fjob_args_str = "" + + try: + metadata_dir = ci_lib.get_ci_metadata_dir() + fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" + if not fournos_fjob_path.exists(): + return fjob_project, fjob_args_str + + with open(fournos_fjob_path, encoding="utf-8") as f: + fjob_data = yaml.safe_load(f) + + display_name = fjob_data.get("spec", {}).get("displayName", "") + if not display_name: + return fjob_project, fjob_args_str + + parts = display_name.split() + if not parts: + return fjob_project, fjob_args_str + + fjob_project = parts[0] + fjob_args_str = " ".join(parts[1:]) if len(parts) > 1 else "" + except Exception as e: + logger.warning(f"Failed to read fournos job for project/args: {e}") + + if fjob_args_str: + fjob_args_str = f"with `{fjob_args_str}`" + + return fjob_project, fjob_args_str + + +def _extract_finish_reason_from_status(status: dict[str, Any]) -> str: + """Extract finish reason from status.""" + if not status.get("success", False): + return "failed" + elif status.get("censoring_occurred", False): + return "completed with censoring" + else: + return "completed" + + +def _build_enhanced_notification( + artifact_dir: Path, + project: str, + finish_reason: str, + status: dict[str, Any], +) -> tuple[str, bool]: + """Build enhanced notification with fournos job config and artifact links.""" + fjob_project, fjob_args_str = _get_project_and_args(project) + + status_emoji = "✅" if status.get("success", False) else "❌" + if status.get("censoring_occurred", False): + status_emoji = "⚠️" + + if finish_reason == "failed": + status_emoji = "❌" + + base_status = f"**{status_emoji} Execution of `{fjob_project}` {fjob_args_str} {status_emoji}**" + notification_parts = [base_status] + + # Add job abort message right below overall status if applicable + shutdown_status = status.get("job_shutdown") + if shutdown_status and shutdown_status.get("is_aborted"): + shutdown_value = shutdown_status.get("shutdown_value", "Stop") + notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") + + notification_parts.append("---") + notification_parts.append("") + + execution_engine_config = _get_execution_engine_config() + if execution_engine_config: + notification_parts.append("**Execution Engine Configuration**") + notification_parts.append(execution_engine_config) + + notification_success = True + try: + artifact_links, mlflow_run_url = _extract_artifact_links(status) + + test_status_section = _extract_test_status_section(status) + step_status = _get_step_status_section(artifact_dir, mlflow_run_url) + postprocess_status_links = _get_postprocess_status_links(artifact_dir, mlflow_run_url) + + if test_status_section: + notification_parts.append("") + notification_parts.append("---") + notification_parts.extend(test_status_section) + + if artifact_links: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Artifact Links**") + notification_parts.extend([f"* {link}" for link in artifact_links]) + else: + notification_parts.append("**Artifact Links:** No direct links available") + + if step_status: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Step details**") + for link in step_status: + notification_parts.append(link) + + if postprocess_status_links: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Post-processing Status** ✅") + notification_parts.extend(postprocess_status_links) + + except Exception as e: + logger.exception(f"Failed to build the extended notifications: {e}") + notification_parts.append("**Artifact Links:** Error extracting links") + notification_success = False + + notification_parts.append("") + notification_parts.append("---") + + return "\n".join(notification_parts), notification_success + + +def _get_execution_engine_config() -> str | None: + """Get execution engine configuration for notification.""" + try: + cluster_config = config.project.get_config("cluster", None, warn=False) + if not cluster_config: + return None + + config_parts = [] + for key, value in cluster_config.items(): + if value: + config_parts.append(f"`{key}`: {value}") + + if config_parts: + return "* " + " \n* ".join(config_parts) + except Exception: + pass + return None + + +def _extract_test_status_section(status: dict[str, Any]) -> list[str] | None: + """Extract test status section from status.""" + test_phase = status.get("test_phase") + if not test_phase: + return None + + phase = test_phase.get("phase", "").upper() + message = test_phase.get("message", "") + test_status_emoji = "✅" if phase == "PASSED" else "❌" if phase == "FAILED" else "⚠️" + + return [ + f"**{test_status_emoji} Test Status: {phase} {test_status_emoji}**", + f"**Message:** {message}", + ] + + +def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | None) -> list[str]: + """Get step status section with links.""" + if not artifact_dir or not artifact_dir.exists(): + return [] + + step_status = [] + for step_dir in sorted(artifact_dir.glob("*")): + if not step_dir.is_dir() or step_dir.name.startswith("."): + continue + + step_name = step_dir.name + + # Skip special directories + if step_name == CI_METADATA_DIRNAME: + continue + + exit_status_file = step_dir / CI_METADATA_DIRNAME / "exit_status.yaml" + exit_status_emoji = "❓" + + if exit_status_file.exists(): + try: + with open(exit_status_file, encoding="utf-8") as f: + exit_data = yaml.safe_load(f) + exit_code = exit_data.get("return_code", 999) + if exit_code == 0: + exit_status_emoji = "✅" + else: + exit_status_emoji = "❌" + except Exception: + exit_status_emoji = "❓" + + # Count ERROR and WARNING messages in run.log + log_counts = _count_log_messages(step_dir) + log_summary = _format_log_summary(log_counts) + + mlflow_log_url = ( + f"{mlflow_run_url}/artifacts/{step_name}/run.log" if mlflow_run_url else "#" + ) + if mlflow_run_url: + step_title = f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}){log_summary}" + step_status.append(step_title) + + step_status.extend(_process_notification_files(step_dir)) + + step_details = _process_step_details(step_dir, mlflow_run_url) + if step_details: + step_status.extend(step_details) + + return step_status + + +def _count_log_messages(step_dir: Path) -> dict[str, int]: + """Count ERROR and WARNING messages in run.log file. + + Args: + step_dir: Directory containing the run.log file + + Returns: + Dict with 'errors' and 'warnings' counts + """ + log_file = step_dir / "run.log" + counts = {"errors": 0, "warnings": 0} + + if not log_file.exists(): + return counts + + try: + with open(log_file, encoding="utf-8", errors="ignore") as f: + for line in f: + line = line.strip() + if line.startswith("ERROR:"): + counts["errors"] += 1 + elif line.startswith("WARNING:"): + counts["warnings"] += 1 + except Exception as e: + logger.warning(f"Failed to read log file {log_file}: {e}") + + return counts + + +def _format_log_summary(log_counts: dict[str, int]) -> str: + """Format log counts for display in step title. + + Args: + log_counts: Dict with 'errors' and 'warnings' counts + + Returns: + Formatted string to append to step title, or empty string if no issues + """ + errors = log_counts.get("errors", 0) + warnings = log_counts.get("warnings", 0) + + if errors == 0 and warnings == 0: + return "" + + parts = [] + if errors > 0: + parts.append(f"🔴 {errors}E") + if warnings > 0: + parts.append(f"🟡 {warnings}W") + + return f" ({', '.join(parts)})" + + +def _get_postprocess_status_links( + artifact_dir: Path | None, mlflow_run_url: str | None +) -> list[str]: + """Get postprocess status links.""" + if not artifact_dir or not artifact_dir.exists(): + return [] + + step_log_links = [] + + # Look for postprocess results in step directories + for step_dir in sorted(artifact_dir.glob("*")): + if not step_dir.is_dir() or step_dir.name.startswith("."): + continue + + step_name = step_dir.name + + # Skip special directories + if step_name == CI_METADATA_DIRNAME: + continue + + try: + postprocess_status_file = step_dir / "postprocess_status.yaml" + if not postprocess_status_file.exists(): + continue + + with open(postprocess_status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f.read()) + + if not status_data: + continue + + # Add job shutdown status if available + if "job_shutdown" in status_data: + shutdown_status = status_data["job_shutdown"] + status_data["job_shutdown"] = shutdown_status + + # Import notification functions from caliper + from projects.caliper.orchestration.notification import ( + format_postprocess_status_notification, + parse_postprocess_result, + ) + + # Parse postprocess result + result = parse_postprocess_status(status_data) + if not result: + continue + + # Create file link function for this step + def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: + if mlflow_run_url: + # Create MLflow artifact URL + return _create_mlflow_file_url_for_step( + mlflow_run_url, step_subdir, str(file_path) + ) + + # Generate notification text from the structured result + notification_text = format_postprocess_status_notification(result, get_file_link) + if notification_text: + step_log_links.append(notification_text) + + except Exception as e: + logger.exception(f"Failed to process postprocess status for {step_name}: {e}") + + return step_log_links + + +def _extract_artifact_links(status: dict[str, Any]) -> tuple[list[str], str | None]: + """Extract artifact links and MLflow URL from status.""" + artifact_links = [] + mlflow_run_url = None + + caliper_export = status.get("caliper_artifacts_export", {}) + backends = caliper_export.get("backends", {}) + + for backend_name, backend_result in backends.items(): + if not isinstance(backend_result, dict): + continue + + if backend_result.get("experiment_url"): + artifact_links.append( + f"[{backend_name} Experiment]({backend_result['experiment_url']})" + ) + + if backend_result.get("run_url"): + mlflow_run_url = backend_result["run_url"] + artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") + elif backend_result.get("artifact_url"): + artifact_links.append(f"[{backend_name} Artifacts]({backend_result['artifact_url']})") + elif backend_result.get("dashboard_url"): + artifact_links.append(f"[{backend_name} Dashboard]({backend_result['dashboard_url']})") + + if status.get("artifact_url"): + artifact_links.append(f"[Artifacts]({status['artifact_url']})") + + return artifact_links, mlflow_run_url + + +def _create_mlflow_file_url_for_step( + mlflow_run_url: str, step_dir_name: str, file_path: str +) -> str: + """Create MLflow URL for a specific file within a step directory. + + Args: + mlflow_run_url: Base MLflow run URL + step_dir_name: Name of the step directory + file_path: Relative path to file from step directory + + Returns: + Full MLflow URL to the file + + Raises: + ValueError: If URL format is unexpected + """ + if "/artifacts" not in mlflow_run_url: + raise ValueError(f"Unexpected MLflow URL format: {mlflow_run_url}") + + # Clean file path + file_clean = file_path.lstrip("/") + + if "#" in mlflow_run_url: + base_domain, hash_fragment = mlflow_run_url.split("#", 1) + if "/artifacts" not in hash_fragment: + raise ValueError("Artifacts not found in hash fragment") + + hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) + # Extract workspace parameter if present, ignoring existing path + workspace_param = "" + if "?workspace=" in artifacts_part: + workspace_param = artifacts_part[artifacts_part.find("?") :] + return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" + else: + base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) + # Extract workspace parameter if present, ignoring existing path + workspace_param = "" + if "?workspace=" in artifacts_part: + workspace_param = artifacts_part[artifacts_part.find("?") :] + return f"{base_url}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" + + +def _process_notification_files(step_dir: Path) -> list[str]: + """Process notification files from step directory.""" + notifications_dir = step_dir / CI_METADATA_DIRNAME / "notifications" + if not (notifications_dir.exists() and notifications_dir.is_dir()): + return [] + + notifications_from_files = [] + for notification_file in sorted(notifications_dir.glob("*.txt")): + with open(notification_file, encoding="utf-8") as f: + content = f.read().strip() + + if not content: + continue + + subtitle = notification_file.stem.replace("__", " ").replace("_", " ").title() + subtitle = re.sub(r"^\d+\s+", "", subtitle) + notifications_from_files.append(f"##### {subtitle}") + + for line in content.splitlines(): + notifications_from_files.append(f"> {line}") + + return notifications_from_files + + +def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: + """Extract test execution information from __test_labels__.yaml files. + + Args: + artifact_dir: Directory to search for __test_labels__.yaml files + mlflow_run_url: Optional MLflow run URL for creating links + + Returns: + List of formatted strings with test information (directory, labels, success, message) + """ + test_info_lines = [] + + # Search for __test_labels__.yaml files recursively + test_labels_files = list(artifact_dir.glob("**/__test_labels__.yaml")) + + if not test_labels_files: + return [] + + for test_labels_file in test_labels_files: + try: + with open(test_labels_file, encoding="utf-8") as f: + test_data = yaml.safe_load(f) or {} + + # Extract directory relative to artifact_dir - use just the immediate directory name + relative_dir = test_labels_file.parent.relative_to(artifact_dir) + dir_name = relative_dir.name if relative_dir != Path(".") else "root" + + # Extract completion info + completion = test_data.get("completion", {}) + success = completion.get("success") + message = completion.get("message") + + if message: + message = f": `{message}`" + + # Format status + if success: + status_emoji = "✅" + elif success is False: + status_emoji = "❌" + else: + status_emoji = "❓" + + # Create link to __test_labels__.yaml file if MLflow URL is available + if mlflow_run_url: + try: + # Get the step directory name (relative to parent) + step_dir_name = str(relative_dir) + test_labels_url = _create_mlflow_file_url_for_step( + mlflow_run_url, step_dir_name, "__test_labels__.yaml" + ) + dir_link = f"[**{dir_name}**]({test_labels_url})" + except Exception as e: + logger.warning(f"Failed to create MLflow link for {test_labels_file}: {e}") + dir_link = f"**{dir_name}**" + else: + dir_link = f"**{dir_name}**" + + test_info_lines.append(f"* {status_emoji} {dir_link}{message}") + + except Exception as e: + test_info_lines.append(f"**{test_labels_file.name}**: Error reading file - {e}") + + return test_info_lines + + +def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: + """Extract post-processing status information from postprocess_status.yaml files. + + Returns: + List of formatted strings with postprocess step status (success only, no details) + """ + postprocess_info_lines = [] + + # Search for postprocess_status.yaml files recursively + postprocess_files = list(artifact_dir.glob("**/postprocess_status.yaml")) + + if not postprocess_files: + return [] + + for postprocess_file in postprocess_files: + try: + with open(postprocess_file, encoding="utf-8") as f: + postprocess_data = yaml.safe_load(f) or {} + + # Extract directory relative to artifact_dir + relative_dir = postprocess_file.parent.relative_to(artifact_dir) + dir_name = str(relative_dir) if relative_dir != Path(".") else "root" + + # Extract overall status + overall_success = postprocess_data.get("success", False) + final_status = postprocess_data.get("final_status", "unknown") + + # Extract individual step statuses + steps = postprocess_data.get("steps", []) + step_statuses = [] + + for step_dict in steps: + for step_name, step_data in step_dict.items(): + if isinstance(step_data, dict): + status = step_data.get("status", "unknown") + status_emoji = ( + "✅" if status == "success" else "❌" if status == "failed" else "⚪" + ) + step_statuses.append(f"{step_name}:{status_emoji}") + + # Format overall line + overall_emoji = "✅" if overall_success else "❌" + if step_statuses: + steps_str = " " + " ".join(step_statuses) + else: + steps_str = f" {final_status}" + + postprocess_info_lines.append(f"**{dir_name}**: {overall_emoji}{steps_str}") + + except Exception as e: + postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") + + return postprocess_info_lines + + +def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> list[str]: + """Process test labels and postprocess status for a single step directory.""" + step_details = [] + + # Extract test labels for this specific step + try: + test_labels_info = _extract_test_labels_info(step_dir, mlflow_run_url) + if test_labels_info: + step_details.extend(test_labels_info) + except Exception as e: + logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") + + # Extract postprocess status for this specific step + try: + postprocess_info = _extract_postprocess_status_info(step_dir) + if postprocess_info: + step_details.extend(postprocess_info) + except Exception as e: + logger.warning(f"Failed to extract postprocess status for step {step_dir.name}: {e}") + + return step_details + + +def _check_job_shutdown_status() -> dict[str, Any] | None: + """Check if the job has been aborted via spec.shutdown field.""" + try: + from projects.core.library import ci as ci_lib + + metadata_dir = ci_lib.get_ci_metadata_dir() + fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" + if not fournos_fjob_path.exists(): + return None + + with open(fournos_fjob_path, encoding="utf-8") as f: + fjob_data = yaml.safe_load(f) + + shutdown_value = fjob_data.get("spec", {}).get("shutdown") + if shutdown_value: + return { + "shutdown_detected": True, + "shutdown_value": shutdown_value, + "is_aborted": shutdown_value.lower() == "stop", + } + + return {"shutdown_detected": False, "shutdown_value": None, "is_aborted": False} + except Exception as e: + logger.warning(f"Failed to check job shutdown status: {e}") + return None + + +def _read_step_exit_status( + step_dir: Path, current_step_name: str | None = None +) -> tuple[str, StepStatus]: + """Read exit status from step directory and return emoji and status enum.""" + from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME + + try: + exit_status_file = step_dir / CI_METADATA_DIRNAME / "exit_status.yaml" + if not exit_status_file.exists(): + # Check if this is the current ongoing step + if current_step_name and step_dir.name == current_step_name: + return "🔄", StepStatus.ONGOING # Ongoing step + return "❓", StepStatus.UNKNOWN # Unknown status if file doesn't exist + + with open(exit_status_file, encoding="utf-8") as f: + exit_data = yaml.safe_load(f) + + return_code = exit_data.get("return_code") + if return_code is None or return_code == 0: + return "✅", StepStatus.SUCCESS + else: + return "❌", StepStatus.FAILURE + except Exception as e: + logger.warning(f"Failed to read exit status from {step_dir}: {e}") + # Check if this is the current ongoing step even on error + if current_step_name and step_dir.name == current_step_name: + return "🔄", StepStatus.ONGOING # Ongoing step + return "❓", StepStatus.UNKNOWN # Unknown status on error + + +def _check_postprocess_warnings(step_dir: Path) -> StepStatus: + """Check for warning status in postprocess status file.""" + + status = StepStatus.SUCCESS # No postprocess warning/error, assume no warnings + for status_file in step_dir.glob("**/postprocess_status.yaml"): + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + except Exception as e: + logging.error(f"Failed to read {status_file} as yaml: {e}") + status = StepStatus.WARNING + continue + + if not status_data: + continue + + # Check top-level success field for warning value + success_value = status_data.get("success") + if success_value == "warning": + logging.warning( + f"Post-process warning detected in {status_file}, setting the WARNING flag" + ) + status = StepStatus.WARNING + + if success_value in ("failure", "error"): + logging.error(f"Post-process {success_value} detected, raising the FAILURE flag") + return StepStatus.FAILURE + + return status + + +def _get_overall_status_from_steps(artifact_dir: Path) -> str: + """Check all step exit statuses and return overall status emoji.""" + from projects.core.library import env + from projects.core.library.export import StepStatus + + try: + current_step_name = Path(env.BASE_ARTIFACT_DIR).name + + step_statuses = [] + + for step_dir in sorted(artifact_dir.iterdir()): + if not step_dir.is_dir(): + continue + if step_dir.name.startswith("."): + continue + + # Only check directories that have run.log (actual steps) + run_log = step_dir / "run.log" + if not run_log.exists(): + continue + + _emoji, status = _read_step_exit_status(step_dir, current_step_name) + + step_statuses.append(status) + + # Check for postprocess warnings in this step (always check, regardless of exit status) + postprocess_status = _check_postprocess_warnings(step_dir) + step_statuses.append(postprocess_status) + + # Priority: failure > ongoing > warning > unknown > success + if StepStatus.FAILURE in step_statuses: + return "🔴" # Any failure = red + elif StepStatus.WARNING in step_statuses: + return "🟠" # Warning = orange + elif StepStatus.UNKNOWN in step_statuses: + return "🟠" # Unknown = orange + elif StepStatus.ONGOING in step_statuses: + return "🟢" # Ongoing --> success + else: + return "🟢" # All successful = green + + except Exception as e: + logger.exception(f"Failed to check step statuses: {e}") + return "🔴" # Error checking = red + + +def _create_mlflow_url(mlflow_run_url: str, step_dir_name: str) -> str | None: + """Create MLflow URL for step logs.""" + + if not mlflow_run_url: + return f"BASE_URL_MISSING/{step_dir_name}" + + if "/artifacts" not in mlflow_run_url: + logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") + return None + + if "#" in mlflow_run_url: + base_domain, hash_fragment = mlflow_run_url.split("#", 1) + if "/artifacts" not in hash_fragment: + raise ValueError("Artifacts not found in hash fragment") + + hash_base, params = hash_fragment.split("/artifacts", 1) + workspace_param = params if "?workspace=" in params else "" + return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/run.log{workspace_param}" + else: + base_url, params = mlflow_run_url.split("/artifacts", 1) + workspace_param = params if "?workspace=" in params else "" + return f"{base_url}/artifacts/{step_dir_name}/run.log{workspace_param}" + + +def _create_mlflow_step_url(mlflow_run_url: str, step_dir_name: str) -> str | None: + """Create MLflow URL for step directory (for file access).""" + if "/artifacts" not in mlflow_run_url: + logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") + return None + + if "#" in mlflow_run_url: + base_domain, hash_fragment = mlflow_run_url.split("#", 1) + if "/artifacts" not in hash_fragment: + raise ValueError("Artifacts not found in hash fragment") + + hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) + # Extract workspace parameter if present, ignoring existing path + workspace_param = "" + if "?workspace=" in artifacts_part: + workspace_param = artifacts_part[artifacts_part.find("?") :] + return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}{workspace_param}" + else: + base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) + # Extract workspace parameter if present, ignoring existing path + workspace_param = "" + if "?workspace=" in artifacts_part: + workspace_param = artifacts_part[artifacts_part.find("?") :] + return f"{base_url}/artifacts/{step_dir_name}{workspace_param}" + + +def _read_step_duration(step_dir: Path) -> str: + """Read step duration from timing file.""" + from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME + + timing_file = step_dir / CI_METADATA_DIRNAME / "test_duration.yaml" + if not timing_file.exists(): + return "" + + try: + with open(timing_file, encoding="utf-8") as f: + timing_data = yaml.safe_load(f) + + formatted_duration = timing_data.get("duration", {}).get("formatted") + return formatted_duration or "" + except Exception as timing_error: + logger.warning(f"Failed to read timing file {timing_file}: {timing_error}") + return "" + + +def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: + """Process step logs from parent directory.""" + from projects.core.library import env + + if not mlflow_run_url: + logging.warning("mlflow_run_url not set. Will generate dummy links.") + + step_status = [] + + current_step_name = Path(env.BASE_ARTIFACT_DIR).name + + for step_dir in sorted(artifact_dir.iterdir()): + if not step_dir.is_dir(): + continue + if step_dir.name.startswith("."): + continue + + run_log = step_dir / "run.log" + if not run_log.exists(): + continue + + mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_dir.name) + if not mlflow_log_url: + mlflow_log_url = "NO_URL" + + step_name = step_dir.name.replace("__", " ").replace("_", " ").title() + duration_str = _read_step_duration(step_dir) + exit_status_emoji, exit_status = _read_step_exit_status(step_dir, current_step_name) + + step_status.append("") + if duration_str: + step_status.append( + f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}) `{duration_str}`" + ) + else: + step_status.append(f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url})") + + step_status.extend(_process_notification_files(step_dir)) + + step_details = _process_step_details(step_dir, mlflow_run_url) + + # Add test execution info for this step + if step_details["test_labels_info"]: + # Add Test Execution Overview if we have any step info + step_status.append("") + step_status.append("**Test Execution Overview**") + + step_status.extend([f"* {info}" for info in step_details["test_labels_info"]]) + + # Add postprocess info for this step + if step_details["postprocess_info"]: + step_status.extend([f"* {info}" for info in step_details["postprocess_info"]]) + + return step_status + + +def _extract_duration_from_status(status: dict[str, Any]) -> str: + """Extract duration from status object.""" + # Look for duration in status + duration = status.get("duration") + if duration: + return f" after {duration}" + return "" diff --git a/projects/core/library/step_status.py b/projects/core/library/step_status.py new file mode 100644 index 000000000..2325385d9 --- /dev/null +++ b/projects/core/library/step_status.py @@ -0,0 +1,15 @@ +"""Step status enumeration for export operations.""" + +from __future__ import annotations + +from enum import StrEnum + + +class StepStatus(StrEnum): + """Status of a step execution.""" + + SUCCESS = "success" + FAILURE = "failure" + ONGOING = "ongoing" + UNKNOWN = "unknown" + WARNING = "warning" From aa90c17594afebfecf75ebb5fb202089d6658adf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:47:57 +0200 Subject: [PATCH 0789/1072] [core] library: export: more flexibility --- projects/core/library/export.py | 100 +++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 16 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 44061096c..eb4cb1c79 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -17,6 +17,8 @@ import yaml from projects.caliper.orchestration.export import ( + CensoringOccurredException, + ExportFailedException, run_from_orchestration_config, ) from projects.core.library import ci as ci_lib @@ -165,7 +167,9 @@ def get_file_link(file_path: str, step_subdir=step_subdir) -> str: raise -def run_caliper_orchestration_export(*, artifact_dir: Path): +def run_caliper_orchestration_export( + *, artifact_dir: Path, disable_censoring: bool = False, disable_file_export: bool = False +): # Use FJOB_NAME as fallback for mlflow run_name if not configured run_name = config.project.get_config( @@ -215,7 +219,9 @@ def run_caliper_orchestration_export(*, artifact_dir: Path): caliper_cfg = config.project.get_config("caliper", print=False) - return run_from_orchestration_config(caliper_cfg) + return run_from_orchestration_config( + caliper_cfg, disable_censoring=disable_censoring, disable_file_export=disable_file_export + ) @click.command("export-artifacts") @@ -233,9 +239,37 @@ def run_caliper_orchestration_export(*, artifact_dir: Path): default=False, help="Show what would be exported and notified without actually performing operations.", ) +@click.option( + "--disable-notification", + "disable_notification", + is_flag=True, + default=False, + help="Skip sending completion notifications.", +) +@click.option( + "--disable-censoring", + "disable_censoring", + is_flag=True, + default=False, + help="Skip censoring sensitive artifacts before export.", +) +@click.option( + "--disable-file-export", + "disable_file_export", + is_flag=True, + default=False, + help="Skip artifact file upload but still run notifications with mock status.", +) @click.pass_context @ci_lib.safe_ci_entrypoint -def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): +def caliper_export_entrypoint( + _ctx, + artifact_dir: Path | None, + dry_run: bool, + disable_notification: bool, + disable_censoring: bool, + disable_file_export: bool, +): """Export the file artifacts.""" notification_provider = getattr(getattr(_ctx, "obj", None), "notification_provider", None) @@ -266,6 +300,9 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): ) return 1 + # Normalize artifact_dir to a pathlib.Path after precedence resolution + artifact_dir = Path(artifact_dir) + if dry_run: logging.info(f"DRY RUN: Building caliper notification from {artifact_dir}") else: @@ -289,9 +326,9 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): "mlflow": { "success": True, "run_id": "dry-run-mock-id", - "experiment_url": "http://localhost:5000/#/experiments/123", - "run_url": "http://localhost:5000/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", - "tracking_uri": "http://localhost:5000", + "experiment_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123", + "run_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", + "tracking_uri": "http://DRY_RUN_MLFLOW_FAKE_URL", } } }, @@ -302,21 +339,43 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): }, } else: - status = run_caliper_orchestration_export(artifact_dir=artifact_dir) + status = run_caliper_orchestration_export( + artifact_dir=artifact_dir, + disable_censoring=disable_censoring, + disable_file_export=disable_file_export, + ) logger.info("Export status:\n" + yaml.dump(status, indent=4)) - # Update fjob status with export results - _update_fjob_export_status(status) - + # Update fjob status with export results (only if file export is not disabled) + if not disable_file_export: + _update_fjob_export_status(status) + else: + logger.info("Skipping fjob status update due to --disable-file-export flag") + + except CensoringOccurredException as e: + logger.info(f"Export completed with censoring: {e}") + # Create success status with censoring flag for notification + status = { + "success": True, + "final_status": "success_with_censoring", + "censoring_occurred": True, + "message": str(e), + "backends": {}, + } + except ExportFailedException as e: + logger.exception(f"Export failed: {e}") + export_failed = True + # Create failure status for notification + status = {"success": False, "error": str(e), "backends": {}} except Exception as e: - logger.error(f"Export failed: {e}") + logger.exception(f"Export failed with unexpected error: {e}") export_failed = True # Create failure status for notification status = {"success": False, "error": str(e), "backends": {}} finally: # Send completion notifications regardless of success/failure - if status: + if status and not disable_notification: try: notification_success = send_notification( artifact_dir, @@ -330,14 +389,23 @@ def caliper_export_entrypoint(_ctx, artifact_dir: Path | None, dry_run: bool): except Exception as e: logger.exception(f"Failed to send notifications: {e}") notification_failed = True + elif disable_notification: + logger.info("Notifications disabled via --disable-notification flag") - if not dry_run: - _update_final_artifacts(artifact_dir, status) + if not disable_file_export: + if not dry_run: + _update_final_artifacts(artifact_dir, status) + else: + logger.info("DRY RUN: Skipping final artifacts update to MLflow") else: - logger.info("DRY RUN: Skipping final artifacts update to MLflow") + logger.info("Skipping final artifacts upload due to --disable-file-export flag") if export_failed or notification_failed: - return 1 + return 1, "failed" + + # Check if censoring occurred and return exit code 1 if so + if status and status.get("censoring_occurred", False): + return 1, "censoring_occurred" return 0 From da7cd85bd3df184b4ae783a7e7909a80ebf7dd32 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:35:33 +0200 Subject: [PATCH 0790/1072] [core] library: vault: expose the sensible flag in the vault --- projects/core/library/vault.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/core/library/vault.py b/projects/core/library/vault.py index 18232702b..da03e2d7d 100644 --- a/projects/core/library/vault.py +++ b/projects/core/library/vault.py @@ -31,6 +31,7 @@ class VaultContent: name: str description: str filename: str | None = None + sensible: bool = True _vault: Optional["VaultDefinition"] = None def __post_init__(self): @@ -38,6 +39,11 @@ def __post_init__(self): if self.filename is None: self.filename = self.name + @property + def is_sensible(self) -> bool: + """Whether this content should be considered sensitive for censoring purposes""" + return self.sensible + @property def file_path(self) -> Path | None: """Get the full absolute path to this content file""" @@ -107,13 +113,15 @@ def _load_vault_definition(self, vault_file: Path) -> VaultDefinition: # New format with file mapping and description filename = content_def.get("file", content_name) description = content_def.get("description", "") # Don't provide default + sensible = content_def.get("sensible", True) # Default to True else: # Legacy format - content_def is the description filename = content_name description = content_def if content_def else "" + sensible = True # Sensible by default content[content_name] = VaultContent( - name=content_name, description=description, filename=filename + name=content_name, description=description, filename=filename, sensible=sensible ) vault_def = VaultDefinition( From 6674ff39972ec8bf96f96dd00e87248bac2ad970 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 12:12:45 +0200 Subject: [PATCH 0791/1072] [core] library: vault: show the stack_info to know the call site --- projects/core/library/vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/vault.py b/projects/core/library/vault.py index da03e2d7d..bf270501e 100644 --- a/projects/core/library/vault.py +++ b/projects/core/library/vault.py @@ -443,7 +443,7 @@ def init( global _vault_manager, _strict_validation_enabled if _vault_manager is not None: - logger.warning("VaultManager already initialized") + logger.warning("VaultManager already initialized", stack_info=True) return _vault_manager = VaultManager() From 494776550a43216f23c6e553c47ff66a5bbb591a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 28 Jul 2026 17:04:18 +0200 Subject: [PATCH 0792/1072] [core] library: config: optimize the IO --- projects/core/library/config.py | 91 +++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index b4d99ebd7..f5f029ba9 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -166,40 +166,60 @@ def apply_config_overrides( logger.fatal(msg) raise ValueError(msg) - for key, value in variable_overrides.items(): - MAGIC_DEFAULT_VALUE = object() - handled_secretly = True # current_value MUST NOT be printed below. - current_value = self.get_config( - key, - MAGIC_DEFAULT_VALUE, - print=False, - warn=False, - handled_secretly=handled_secretly, - ) - if current_value == MAGIC_DEFAULT_VALUE: - try: - # Try to create the key if parent exists and is a dict - self._create_first_parent_config_key(key, value) - self.save_config() - except ValueError: - if not ignore_not_found: - raise + # Setup presets_applied.txt file for writing variable overrides + dest_txt = env.ARTIFACT_DIR / CI_METADATA_DIRNAME / "presets_applied.txt" + dest_txt.parent.mkdir(parents=True, exist_ok=True) + + # Collect all override messages to write to file once + file_messages = [] + try: + for key, value in variable_overrides.items(): + MAGIC_DEFAULT_VALUE = object() + handled_secretly = True # current_value MUST NOT be printed below. + current_value = self.get_config( + key, + MAGIC_DEFAULT_VALUE, + print=False, + warn=False, + handled_secretly=handled_secretly, + ) + if current_value == MAGIC_DEFAULT_VALUE: + try: + # Try to create the key if parent exists and is a dict + self._create_first_parent_config_key(key, value) + self.save_config() + except ValueError: + if not ignore_not_found: + raise + + if log: + msg = f"config override IGNORED: {key} --> {value}" + logger.info(msg) + file_messages.append(msg) + continue + + self.save_config() if log: - logger.info(f"config override IGNORED: {key} --> {value}") + msg = f"config override (new key): {key} --> {value}" + logger.info(msg) + file_messages.append(msg) continue - self.save_config() + self.set_config(key, value, print=False) + actual_value = self.get_config( + key, print=False + ) # ensure that key has been set, raises an exception otherwise if log: - logger.info(f"config override (new key): {key} --> {value}") - continue - - self.set_config(key, value, print=False) - actual_value = self.get_config( - key, print=False - ) # ensure that key has been set, raises an exception otherwise - if log: - logger.info(f"config override: {key} --> {actual_value}") + msg = f"config override: {key} --> {actual_value}" + logger.info(msg) + file_messages.append(msg) + finally: + # Write all collected messages to file, even if processing failed + if file_messages: + with open(dest_txt, "a") as f: + for msg in file_messages: + print(msg, file=f) def apply_preset(self, name): values = self.get_preset(name) @@ -210,6 +230,9 @@ def apply_preset(self, name): dest_txt = env.ARTIFACT_DIR / CI_METADATA_DIRNAME / "presets_applied.txt" dest_txt.parent.mkdir(parents=True, exist_ok=True) + # Collect preset messages to write to file once + preset_messages = [] + for key, value in values.items(): if key == "extends": for extend_name in value or []: @@ -218,12 +241,16 @@ def apply_preset(self, name): msg = f"preset[{name}] {key} --> {value}" logger.info(msg) - - with open(dest_txt, "a") as f: - print(msg, file=f) + preset_messages.append(msg) self.set_config(key, value, print=False) + # Write all collected preset messages to file once + if preset_messages: + with open(dest_txt, "a") as f: + for msg in preset_messages: + print(msg, file=f) + def load_presets(self, preset_dir): for preset_file in preset_dir.glob("*.yaml"): with open(preset_file) as preset_f: From ab474c5a694e2cffa7319913b1bab5ac281074ee Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:39:26 +0200 Subject: [PATCH 0793/1072] [caliper] Add the artifact censoring command --- projects/caliper/cli/commands.py | 172 +++++++++- projects/caliper/cli/main.py | 2 + .../caliper/engine/file_export/censoring.py | 302 ++++++++++++++++++ .../engine/file_export/censoring_rules.py | 106 ++++++ 4 files changed, 575 insertions(+), 7 deletions(-) create mode 100644 projects/caliper/engine/file_export/censoring.py create mode 100644 projects/caliper/engine/file_export/censoring_rules.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 266d2fb2c..90d3cc3f8 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -13,6 +13,7 @@ from projects.caliper.engine.constants import METADATA_FILE from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import +from projects.caliper.engine.file_export.censoring import apply_censoring_to_artifacts from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml from projects.caliper.engine.kpi.generate import run_kpi_generate from projects.caliper.engine.kpi.import_export import ( @@ -1322,15 +1323,17 @@ def artifacts_export( } try: - run_artifacts_export( + exit_code = run_artifacts_export( from_path=from_path, backend=list(backend) if backend else ["mlflow"], - mlflow_tracking_uri=mlflow_tracking_uri, - mlflow_experiment=mlflow_experiment, - mlflow_run_id=mlflow_run_id, - mlflow_run_name=mlflow_run_name, - mlflow_insecure_tls=mlflow_insecure_tls, - mlflow_secrets_path=mlflow_secrets_path, + mlflow_tracking_uri=final_config["tracking_uri"], + mlflow_experiment=final_config["experiment"], + mlflow_run_id=final_config["run_id"], + mlflow_run_name=final_config["run_name"], + mlflow_insecure_tls=final_config["insecure_tls"], + mlflow_secrets_path=Path(final_config["secrets_path"]) + if final_config["secrets_path"] + else None, mlflow_config_data=final_config, dry_run=dry_run, verbose=verbose, @@ -1346,6 +1349,161 @@ def artifacts_export( click.echo(full_traceback, err=True) sys.exit(3) + # Check return value outside try block to avoid intercepting SystemExit + if exit_code != 0: + sys.exit(exit_code) + + +@click.command("censor") +@click.option( + "--from", + "from_path", + type=click.Path(path_type=Path, exists=True), + required=True, + help="Source directory containing artifacts to censor.", +) +@click.option( + "--output", + "output_path", + type=click.Path(path_type=Path), + default=None, + help="Output directory for cleaned artifacts. If not specified, censoring is done in-place.", +) +@click.option("--dry-run", is_flag=True, help="Show what would be censored without making changes.") +@click.option("-v", "--verbose", is_flag=True, help="Show detailed censoring information.") +@click.option( + "--report", + "report_path", + type=click.Path(path_type=Path), + default=None, + help="Write censoring report to this file.", +) +def artifacts_censor( + from_path: Path, + output_path: Path | None, + dry_run: bool, + verbose: bool, + report_path: Path | None, +) -> None: + """Censor sensitive artifacts by removing files containing secrets or sensitive patterns.""" + import json + import shutil + from datetime import datetime + + if not from_path.exists(): + click.echo(f"Error: Source path does not exist: {from_path}", err=True) + sys.exit(1) + + if not from_path.is_dir(): + click.echo(f"Error: Source path is not a directory: {from_path}", err=True) + sys.exit(1) + + # Collect all artifact files + all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] + + if not all_artifact_paths: + click.echo(f"No artifacts found in {from_path}") + return + + click.echo(f"Scanning {len(all_artifact_paths)} artifacts in {from_path}") + + # Apply censoring + filtered_paths, censoring_results = apply_censoring_to_artifacts( + all_artifact_paths, censoring_enabled=True, verbose=verbose, dry_run=dry_run + ) + + # Separate censored and clean files + censored_files = [r.file_path for r in censoring_results if r.censored] + clean_files = [r.file_path for r in censoring_results if not r.censored] + + # Print summary + click.echo("\nCensoring Summary:") + click.echo(f" Total files scanned: {len(all_artifact_paths)}") + click.echo(f" Clean files: {len(clean_files)}") + click.echo(f" Censored files: {len(censored_files)}") + + if censored_files and verbose: + click.echo("\nCensored files:") + for result in censoring_results: + if result.censored: + rel_path = result.file_path.relative_to(from_path) + click.echo(f" - {rel_path}: {result.reason}") + + # Generate report if requested + if report_path: + report_data = { + "timestamp": datetime.now().isoformat(), + "source_directory": str(from_path), + "output_directory": str(output_path) if output_path else None, + "total_files": len(all_artifact_paths), + "clean_files": len(clean_files), + "censored_files": len(censored_files), + "censored_details": [ + { + "file": str(r.file_path.relative_to(from_path)), + "reason": r.reason, + } + for r in censoring_results + if r.censored + ], + } + + if dry_run: + click.echo(f"\nDry run: Would write report to {report_path}") + else: + try: + report_path.parent.mkdir(parents=True, exist_ok=True) + with open(report_path, "w") as f: + json.dump(report_data, f, indent=2) + click.echo( + f"\nCensoring report written to {report_path}\n{report_path.read_text()}" + ) + except Exception as e: + click.echo(f"Error writing report: {e}", err=True) + + # Handle output + if dry_run: + click.echo("\nDry run: No files were modified") + if output_path: + click.echo(f"Would copy {len(clean_files)} clean files to {output_path}") + else: + click.echo(f"Would remove {len(censored_files)} censored files from {from_path}") + + elif output_path: + # Copy clean files to output directory + try: + output_path.mkdir(parents=True, exist_ok=True) + + for clean_file in clean_files: + rel_path = clean_file.relative_to(from_path) + dest_path = output_path / rel_path + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(clean_file, dest_path) + + click.echo(f"\nCopied {len(clean_files)} clean files to {output_path}") + + except Exception as e: + click.echo(f"Error copying files: {e}", err=True) + sys.exit(1) + + else: + # Remove censored files in-place + if censored_files: + try: + for censored_file in censored_files: + censored_file.unlink() + if verbose: + rel_path = censored_file.relative_to(from_path) + click.echo(f" Removed: {rel_path}") + + click.echo(f"\nRemoved {len(censored_files)} censored files from {from_path}") + + except Exception as e: + click.echo(f"Error removing files: {e}", err=True) + sys.exit(1) + else: + click.echo("\nNo files to remove - all artifacts are clean!") + @click.command("import") @click.option("--from-mlflow", "mlflow_run_id", help="MLflow run ID to download artifacts from.") diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 34acb10af..49904c231 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -12,6 +12,7 @@ from projects.caliper.cli.commands import ( ai_eval_export, analyse_kpis_cmd, + artifacts_censor, artifacts_export, artifacts_import, html_export_cmd, @@ -221,6 +222,7 @@ def run_cli() -> None: kpi_group.add_command(s3_export_cmd) # Register artifacts commands +artifacts_group.add_command(artifacts_censor) artifacts_group.add_command(artifacts_export) artifacts_group.add_command(artifacts_import) diff --git a/projects/caliper/engine/file_export/censoring.py b/projects/caliper/engine/file_export/censoring.py new file mode 100644 index 000000000..b8218b764 --- /dev/null +++ b/projects/caliper/engine/file_export/censoring.py @@ -0,0 +1,302 @@ +""" +Artifact censoring module for Caliper. + +This module provides censoring capabilities to filter artifacts before upload: +1. Keyword-based censoring - filters files containing specified keywords/patterns +2. Filename-based censoring - filters files with sensitive filename patterns +""" + +from __future__ import annotations + +import logging +import stat +from dataclasses import dataclass +from pathlib import Path + +from .censoring_rules import ( + COMPILED_KEYWORD_PATTERNS, + matches_sensitive_filename, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class CensoringResult: + """Result of censoring operation on a single file.""" + + file_path: Path + censored: bool + reason: str + sanitized: bool = False # True if content was sanitized in-place + + def __str__(self): + if self.sanitized: + return f"SANITIZED: {self.file_path} ({self.reason})" + elif self.censored: + return f"EXCLUDED: {self.file_path} ({self.reason})" + else: + return f"ALLOWED: {self.file_path} ({self.reason})" + + +class ArtifactCensor: + """Main censoring class that applies keyword and secret filtering.""" + + def __init__( + self, + vault_secrets: set[str] | None = None, + secret_mapping: dict[str, str] | None = None, + verbose: bool = False, + dry_run: bool = False, + ): + """ + Initialize the artifact censor. + + Args: + vault_secrets: Set of secret strings loaded from vaults + secret_mapping: Dict mapping secret strings to vault/content identifiers + verbose: Enable verbose logging + dry_run: Skip file modifications while preserving analysis + """ + self.vault_secrets = vault_secrets or set() + self.secret_mapping = secret_mapping or {} + self.verbose = verbose + self.dry_run = dry_run + + if self.verbose: + logger.info(f"Initialized censoring with {len(self.vault_secrets)} vault secrets") + + def _is_text_file(self, file_path: Path) -> bool: + """Check if file is likely a text file based on extension and content sample.""" + # Common text file extensions + text_extensions = { + ".txt", + ".log", + ".yaml", + ".yml", + ".json", + ".xml", + ".csv", + ".md", + ".rst", + ".py", + ".js", + ".html", + ".css", + ".sql", + ".sh", + ".bash", + ".conf", + ".cfg", + ".ini", + ".properties", + ".out", + ".err", + } + + if file_path.suffix.lower() in text_extensions: + return True + + # For files without clear extension, try to detect if it's text + try: + # Read only first 1KB to avoid loading large files into memory + with open(file_path, "rb") as f: + sample = f.read(1024) + if not sample: + return True # Empty file is text + + # Check if most bytes are printable ASCII or common UTF-8 + printable_ratio = sum(1 for b in sample if 32 <= b <= 126 or b in (9, 10, 13)) / len( + sample + ) + return printable_ratio > 0.7 + + except Exception: + return False # If we can't read it, assume it's binary + + def _sanitize_file_content(self, file_path: Path) -> CensoringResult: + """Sanitize file content by replacing sensitive patterns.""" + try: + # First check if filename itself is sensitive - sanitize these files + if matches_sensitive_filename(str(file_path)): + # Replace content with censoring message + censored_content = ( + f"Content censored by caliper - sensitive filename: {file_path.name}\n" + ) + + # Write sanitized content back to file (skip if dry run) + if not self.dry_run: + file_path.write_text(censored_content, encoding="utf-8") + + return CensoringResult( + file_path, True, f"sensitive filename pattern: {file_path.name}", sanitized=True + ) + + # Skip non-text files to avoid reading binary content + if not self._is_text_file(file_path): + return CensoringResult(file_path, False, "non-text file") + + content = file_path.read_text(encoding="utf-8", errors="ignore") + sanitized = False + reasons = [] + + # Import KEYWORD_PATTERNS for reporting + from .censoring_rules import KEYWORD_PATTERNS + + # Check for keyword patterns - redact matched spans in place + keyword_detected = False + matched_patterns = set() + for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + matches = list(pattern.finditer(content)) + if matches: + keyword_detected = True + matched_patterns.add(i) + + if keyword_detected: + # Replace all matched spans with redacted text, preserving other content + # Process patterns in reverse order by position to maintain string indices + all_matches = [] + for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + for match in pattern.finditer(content): + all_matches.append((match.start(), match.end(), i)) + + # Sort by start position in reverse order + all_matches.sort(reverse=True) + + # Replace each match with [REDACTED] + for start, end, _pattern_idx in all_matches: + content = content[:start] + "[REDACTED]" + content[end:] + + # Add reasons for all matched patterns + for pattern_idx in sorted(matched_patterns): + reasons.append(f"contains keyword pattern: {KEYWORD_PATTERNS[pattern_idx]}") + + sanitized = True + + # Always check for vault secrets, independent of keyword pattern detection + for secret in self.vault_secrets: + if secret and secret.strip() and secret.strip() in content: + content = content.replace(secret.strip(), "*******") + sanitized = True + vault_identifier = self.secret_mapping.get(secret.strip(), "unknown vault") + reasons.append(f"contains vault secret: {vault_identifier}") + + if sanitized: + # Write sanitized content back to original file (skip if dry run) + if not self.dry_run: + try: + file_path.write_text(content, encoding="utf-8") + except PermissionError: + # Handle read-only files by making them writable + try: + # Make file writable + file_path.chmod(file_path.stat().st_mode | stat.S_IWUSR) + file_path.write_text(content, encoding="utf-8") + except Exception: + # If we still can't write, abort this file export + raise + + reason = reasons[0] if reasons else "sensitive content detected" + return CensoringResult(file_path, True, reason, sanitized=True) + + return CensoringResult(file_path, False, "content check passed") + + except Exception as e: + logger.warning(f"Error sanitizing file {file_path}: {e}") + return CensoringResult(file_path, True, f"sanitization failed: {e}") + + def censor_files(self, file_paths: list[Path]) -> tuple[list[Path], list[CensoringResult]]: + """ + Apply censoring to a list of file paths, sanitizing content where possible. + + Args: + file_paths: List of file paths to check + + Returns: + tuple: (processed_files, censoring_results) + processed_files contains original paths for clean files, + sanitized paths for files with content replacements, + and excludes files with sensitive filenames + """ + processed_files = [] + results = [] + + for file_path in file_paths: + if not file_path.is_file(): + # Skip directories and non-existent files + processed_files.append(file_path) + continue + + result = self._sanitize_file_content(file_path) + results.append(result) + + if not result.censored: + # Clean file, include original + processed_files.append(file_path) + elif result.sanitized: + # File had content sanitized in-place, include original path + processed_files.append(file_path) + if self.verbose: + logger.info(f"Sanitized: {result}") + else: + # File excluded entirely (e.g., sensitive filename) + if self.verbose: + logger.info(f"Excluded: {result}") + + return processed_files, results + + +def apply_censoring_to_artifacts( + artifact_paths: list[Path], + censoring_enabled: bool = True, + verbose: bool = False, + vault_secrets: set[str] | None = None, + secret_mapping: dict[str, str] | None = None, + dry_run: bool = False, +) -> tuple[list[Path], list[CensoringResult]]: + """ + Apply censoring to artifact paths, sanitizing sensitive content in-place. + + This function processes artifacts by: + - Replacing sensitive content patterns with "Content censored by caliper" in-place + - Replacing vault secrets with "*******" in-place + - Excluding files with sensitive filename patterns (.pem, .key, files with "secret" in name, etc.) + + Args: + artifact_paths: List of artifact file paths + censoring_enabled: Whether to apply censoring + verbose: Enable verbose logging + vault_secrets: Set of vault secret strings to censor + secret_mapping: Dict mapping secret strings to vault/content identifiers + dry_run: Skip file modifications while preserving analysis + + Returns: + tuple: (processed_paths, censoring_results) + processed_paths contains original paths for clean and sanitized files, + and excludes files with sensitive filenames + """ + if not censoring_enabled: + if verbose: + logger.info("Censoring disabled, allowing all artifacts") + return artifact_paths, [] + + # Apply censoring using keyword and filename patterns + censor = ArtifactCensor( + vault_secrets=vault_secrets or set(), + secret_mapping=secret_mapping or {}, + verbose=verbose, + dry_run=dry_run, + ) + processed_paths, results = censor.censor_files(artifact_paths) + + # Log summary + sanitized_count = len([r for r in results if r.sanitized]) + excluded_count = len([r for r in results if r.censored and not r.sanitized]) + clean_count = len([r for r in results if not r.censored]) + + if verbose or sanitized_count > 0 or excluded_count > 0: + logger.info( + f"Censoring complete: {clean_count} clean, {sanitized_count} sanitized, {excluded_count} excluded" + ) + + return processed_paths, results diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py new file mode 100644 index 000000000..242b5d64f --- /dev/null +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -0,0 +1,106 @@ +""" +Censoring rules and patterns for Caliper artifact filtering. + +This module defines the patterns and rules used to identify sensitive content +in artifacts before upload. +""" + +from __future__ import annotations + +import re + +# Keyword patterns to detect in file content +# These are compiled regex patterns that match common sensitive data patterns +KEYWORD_PATTERNS = [ + # Password patterns + r"password\s*[:=]\s*\S+", + r"pwd\s*[:=]\s*\S+", + r"passwd\s*[:=]\s*\S+", + # API key patterns + r"api[_-]?key\s*[:=]\s*\S+", + r"apikey\s*[:=]\s*\S+", + r"api[_-]?secret\s*[:=]\s*\S+", + # Token patterns + r"token\s*[:=]\s*\S+", + r"secret[_-]?token\s*[:=]\s*\S+", + r"access[_-]?token\s*[:=]\s*\S+", + r"refresh[_-]?token\s*[:=]\s*\S+", + # Bearer tokens + r"Bearer\s+[A-Za-z0-9+/=]+", + r"bearer\s+[A-Za-z0-9+/=]+", + # Specific service API keys + r"sk-[a-zA-Z0-9]{32,}", # OpenAI API keys + r"ghp_[a-zA-Z0-9]{36}", # GitHub personal access tokens + r"gho_[a-zA-Z0-9]{36}", # GitHub OAuth tokens + r"ghu_[a-zA-Z0-9]{36}", # GitHub user-to-server tokens + r"ghs_[a-zA-Z0-9]{36}", # GitHub server-to-server tokens + r"ghr_[a-zA-Z0-9]{36}", # GitHub refresh tokens + # AWS patterns + r"AKIA[0-9A-Z]{16}", # AWS Access Key ID + r"aws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*\S+", + # Database connection strings + r"mongodb://[^/\s]+:[^@\s]+@", + r"mysql://[^/\s]+:[^@\s]+@", + r"postgresql://[^/\s]+:[^@\s]+@", + # Generic credential patterns + r"credential\s*[:=]\s*\S+", + r"secret\s*[:=]\s*\S+", + r"private[_-]?key\s*[:=]\s*\S+", +] + +# Compile patterns for better performance +COMPILED_KEYWORD_PATTERNS = [re.compile(pattern, re.IGNORECASE) for pattern in KEYWORD_PATTERNS] + +# File patterns to always censor (by filename) +SENSITIVE_FILE_PATTERNS = [ + r".*\.pem$", # PEM certificate files + r".*\.key$", # Private key files + r".*\.p12$", # PKCS#12 certificate files + r".*\.pfx$", # PKCS#12 certificate files (Windows) + r".*secret.*", # Any file with "secret" in the name + r".*credential.*", # Any file with "credential" in the name + r".*password.*", # Any file with "password" in the name + r".*\.ssh/.*", # SSH directory contents + r".*/\.ssh/.*", # SSH directory contents (with path) + r".*id_rsa.*", # SSH private keys + r".*id_dsa.*", # DSA private keys + r".*id_ecdsa.*", # ECDSA private keys + r".*id_ed25519.*", # Ed25519 private keys + r".*\.env$", # Environment files + r".*\.env\..*", # Environment files with suffixes +] + +# Compile file patterns for better performance +COMPILED_FILE_PATTERNS = [re.compile(pattern, re.IGNORECASE) for pattern in SENSITIVE_FILE_PATTERNS] + + +def matches_sensitive_filename(filename: str) -> bool: + """ + Check if a filename matches any sensitive file pattern. + + Args: + filename: The filename or path to check + + Returns: + bool: True if the filename indicates a sensitive file + """ + return any(pattern.match(filename) for pattern in COMPILED_FILE_PATTERNS) + + +def find_sensitive_content_in_text(content: str) -> list[str]: + """ + Find sensitive content patterns in text. + + Args: + content: Text content to scan + + Returns: + list[str]: List of pattern descriptions that matched + """ + matches = [] + for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + if pattern.search(content): + # Return the original pattern string for logging + matches.append(KEYWORD_PATTERNS[i]) + + return matches From 8db59ac985fe9d958b7200a7fd13f1feb719d2d7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 10:23:06 +0200 Subject: [PATCH 0794/1072] [caliper] engine: file_export: censoring_rules: remove rules to wide --- .../engine/file_export/censoring_rules.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index 242b5d64f..c2b2140a8 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -14,7 +14,6 @@ KEYWORD_PATTERNS = [ # Password patterns r"password\s*[:=]\s*\S+", - r"pwd\s*[:=]\s*\S+", r"passwd\s*[:=]\s*\S+", # API key patterns r"api[_-]?key\s*[:=]\s*\S+", @@ -44,7 +43,6 @@ r"postgresql://[^/\s]+:[^@\s]+@", # Generic credential patterns r"credential\s*[:=]\s*\S+", - r"secret\s*[:=]\s*\S+", r"private[_-]?key\s*[:=]\s*\S+", ] @@ -84,7 +82,30 @@ def matches_sensitive_filename(filename: str) -> bool: Returns: bool: True if the filename indicates a sensitive file """ - return any(pattern.match(filename) for pattern in COMPILED_FILE_PATTERNS) + from pathlib import Path + + # Filename-only patterns that should match just the basename + filename_only_patterns = [ + r".*secret.*", + r".*credential.*", + r".*password.*", + ] + + # Check filename-only patterns against basename + basename = Path(filename).name + for pattern_str in filename_only_patterns: + pattern = re.compile(pattern_str, re.IGNORECASE) + if pattern.match(basename): + return True + + # Check all other patterns against full path + for pattern in COMPILED_FILE_PATTERNS: + pattern_str = pattern.pattern + if pattern_str not in filename_only_patterns: + if pattern.match(filename): + return True + + return False def find_sensitive_content_in_text(content: str) -> list[str]: From bc0c8e4725f53e8a5a42a3cef7a15ae3797d8a78 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:40:56 +0200 Subject: [PATCH 0795/1072] [caliper] orchestration: integrate the censoring --- projects/caliper/orchestration/censoring.py | 318 ++++++++++++++++++ projects/caliper/orchestration/export.py | 76 ++++- projects/core/library/export_notifications.py | 6 +- 3 files changed, 384 insertions(+), 16 deletions(-) create mode 100644 projects/caliper/orchestration/censoring.py diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py new file mode 100644 index 000000000..032c842f9 --- /dev/null +++ b/projects/caliper/orchestration/censoring.py @@ -0,0 +1,318 @@ +""" +Orchestration-level censoring functionality for Caliper artifact export. + +This module provides high-level censoring operations that integrate with the export +orchestration system, including reporting and notification generation. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import yaml + +from projects.caliper.engine.file_export.censoring import apply_censoring_to_artifacts +from projects.caliper.orchestration.export_config import CaliperOrchestrationExportConfig +from projects.core.library import ci as ci_lib +from projects.core.library import env +from projects.core.library import vault as vault_lib + +logger = logging.getLogger(__name__) + +# Vault content types to automatically discover for censoring +VAULT_CONTENT_TYPES = [ + "token", + "password", + "secret", + "key", + "credential", + "tracking_uri", + "username", + "api_key", + "api-key", + "auth", + "authentication", + "bearer", + "oauth", +] + + +def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, str]]: + """ + Discover all vault secrets for censoring (all vault content is treated as sensitive). + + Args: + verbose: Enable verbose logging + + Returns: + Tuple of (secret_strings, secret_mapping) where: + - secret_strings: Set of secret strings loaded from vaults + - secret_mapping: Dict mapping secret string -> vault/content identifier + """ + vault_secrets = set() + secret_mapping = {} + + try: + vault_manager = vault_lib.get_vault_manager() + available_vaults = vault_manager.list_vaults() + + if verbose: + logger.info(f"Discovering vault secrets from {len(available_vaults)} vaults") + + secrets_discovered = 0 + for vault_name in available_vaults: + vault = vault_manager.get_vault(vault_name) + if not vault: + continue + + for content_name, content_def in vault.content.items(): + # Only process content marked as sensible + if not content_def.is_sensible: + if verbose: + logger.info( + f"Skipping non-sensible vault content {vault_name}/{content_name}" + ) + continue + + content_path = vault_manager.get_vault_content_path(vault_name, content_name) + if content_path and content_path.exists(): + try: + # Read vault content (assume it's text) + content_text = content_path.read_text( + encoding="utf-8", errors="ignore" + ).strip() + if content_text: + vault_secrets.add(content_text) + secret_mapping[content_text] = f"{vault_name}/{content_name}" + secrets_discovered += 1 + if verbose: + logger.info( + f"Discovered secret from vault {vault_name}/{content_name}" + ) + except Exception as e: + if verbose: + logger.warning( + f"Failed to read vault content {vault_name}/{content_name}: {e}" + ) + + if verbose: + logger.info(f"Discovered {secrets_discovered} vault secrets for censoring") + + except Exception as e: + logger.warning(f"Failed to discover vault secrets: {e}") + + return vault_secrets, secret_mapping + + +def orchestration_apply_censoring( + from_path: Path, + export_cfg: CaliperOrchestrationExportConfig, + disable_censoring: bool = False, +) -> bool: + """ + Apply censoring for single-run orchestration export. + + Args: + from_path: Source directory containing artifacts + export_cfg: Export configuration + disable_censoring: Whether to skip censoring + + Returns: + True if any files were censored (sanitized or excluded), False otherwise + """ + if disable_censoring: + if export_cfg.verbose: + logger.info("Censoring disabled via --disable-censoring flag") + return False + + # Discover vault secrets + vault_secrets, secret_mapping = discover_vault_secrets(verbose=export_cfg.verbose) + + # Collect all artifact files + all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] + + logger.info( + f"Starting artifact censoring: {len(all_artifact_paths)} files to scan from {from_path}" + ) + + # Apply in-place censoring + processed_paths, censoring_results = apply_censoring_to_artifacts( + all_artifact_paths, + censoring_enabled=True, + verbose=export_cfg.verbose, + vault_secrets=vault_secrets, + secret_mapping=secret_mapping, + ) + + # Separate results by type + clean_files = [r for r in censoring_results if not r.censored] + sanitized_files = [r for r in censoring_results if r.sanitized] + excluded_files = [r for r in censoring_results if r.censored and not r.sanitized] + + logger.info( + f"Censoring complete: {len(clean_files)} clean, {len(sanitized_files)} sanitized, {len(excluded_files)} excluded" + ) + + _generate_censoring_report(censoring_results, from_path) + + # Abort export if any files were excluded (fail closed) + if excluded_files: + # Create notification about censoring activity blocking export + _create_censoring_notification(censoring_results, from_path, export_blocked=True) + + excluded_file_names = [str(r.file_path.relative_to(from_path)) for r in excluded_files] + logger.error( + f"Export aborted: {len(excluded_files)} files contain sensitive content that could not be sanitized: {excluded_file_names[:5]}{'...' if len(excluded_file_names) > 5 else ''}" + ) + raise RuntimeError( + f"Export blocked due to {len(excluded_files)} unsanitizable sensitive files" + ) + + # Generate censoring report and notification if any files were processed + if sanitized_files: + # Create notification about censoring activity (export proceeds with sanitized files) + _create_censoring_notification(censoring_results, from_path, export_blocked=False) + + if export_cfg.verbose: + logger.info( + f"Export proceeding with {len(sanitized_files)} files sanitized for sensitive content" + ) + return True # Censoring occurred + + elif export_cfg.verbose: + logger.info("No files required censoring") + + return False # No censoring occurred + + +def _generate_censoring_report(censoring_results, from_path: Path) -> None: + """Generate a YAML report of censored files in ARTIFACT_DIR.""" + try: + from datetime import datetime + + # Group censored files by reason + censored_by_reason = {} + for r in censoring_results: + if r.censored: + reason = r.reason + if reason not in censored_by_reason: + censored_by_reason[reason] = [] + + file_path = ( + str(r.file_path.relative_to(from_path)) + if r.file_path.is_relative_to(from_path) + else str(r.file_path) + ) + censored_by_reason[reason].append(file_path) + + # Create report data + report_data = { + "timestamp": datetime.now().isoformat(), + "source_directory": str(from_path), + "total_files": len(censoring_results), + "censored_files": len([r for r in censoring_results if r.censored]), + "clean_files": len([r for r in censoring_results if not r.censored]), + "censored_by_reason": censored_by_reason, + } + + # Write report to ARTIFACT_DIR + if env.ARTIFACT_DIR: + report_path = env.ARTIFACT_DIR / "censoring_report.yaml" + report_path.parent.mkdir(parents=True, exist_ok=True) + + with open(report_path, "w") as f: + yaml.dump(report_data, f, indent=2, default_flow_style=False) + + logger.info(f"Censoring report written to {report_path}") + else: + logger.warning("ARTIFACT_DIR not set, cannot write censoring report") + + except Exception as e: + logger.error(f"Failed to generate censoring report: {e}") + + +def _create_censoring_notification( + censoring_results, from_path: Path, export_blocked: bool = False +) -> None: + """Create a notification file about censoring activity.""" + try: + # Separate results + sanitized_files = [r.file_path for r in censoring_results if r.sanitized] + excluded_files = [r.file_path for r in censoring_results if r.censored and not r.sanitized] + + if not sanitized_files and not excluded_files: + return # No censoring occurred + + # Prepare file lists + def format_file_list(files, limit=10): + file_list = "\n".join( + [ + f" - {file.relative_to(from_path) if file.is_relative_to(from_path) else file}" + for file in files[:limit] + ] + ) + if len(files) > limit: + file_list += f"\n ... and {len(files) - limit} more files" + return file_list + + if export_blocked and excluded_files: + # Export blocked due to excluded files + message = f"""Caliper Export Blocked: Sensitive Files Cannot Be Sanitized + +{len(excluded_files)} file(s) with sensitive filenames were excluded from export: + +{format_file_list(excluded_files)} + +These files have filenames that indicate sensitive content and cannot be safely sanitized: +- Certificate files (.pem, .key, .p12, .pfx) +- Files with 'secret', 'credential', or 'password' in their names +- SSH keys and configuration files + +Please review these artifacts and rename or relocate sensitive files before re-running the export. + +For details, see: $ARTIFACT_DIR/censoring_report.yaml""" + + notification_name = "censoring_blocked" + + else: + # Export proceeded with sanitization + total_censored = len(sanitized_files) + len(excluded_files) + message_parts = [ + f"Caliper Export: {total_censored} file(s) contained sensitive content" + ] + + if sanitized_files: + message_parts.append( + f""" +{len(sanitized_files)} file(s) had sensitive content sanitized and included in export: + +{format_file_list(sanitized_files)} + +Sensitive content (passwords, API keys, tokens) was replaced with placeholder text.""" + ) + + if excluded_files: + message_parts.append( + f""" +{len(excluded_files)} file(s) with sensitive filenames were excluded: + +{format_file_list(excluded_files)} + +These files cannot be safely sanitized due to their filenames.""" + ) + + message_parts.append("\nFor details, see: $ARTIFACT_DIR/censoring_report.yaml") + message = "\n".join(message_parts) + "\n" + notification_name = "censoring_applied" + + # Create notification file using CI library + notification_file = ci_lib.add_notification_file(name=notification_name, message=message) + + if notification_file: + logger.info(f"Censoring notification created: {notification_file}") + else: + logger.error("Failed to create censoring notification file") + + except Exception as e: + logger.error(f"Failed to create censoring notification: {e}") diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index b61f31265..5b119d4d6 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -25,6 +25,9 @@ run_multi_run_artifacts_export, ) from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml +from projects.caliper.orchestration.censoring import ( + orchestration_apply_censoring, +) from projects.caliper.orchestration.export_config import ( CaliperOrchestrationExportConfig, ) @@ -35,6 +38,24 @@ logger = logging.getLogger(__name__) +class CaliperExportError(Exception): + """Base exception for Caliper export errors.""" + + pass + + +class CensoringOccurredException(CaliperExportError): + """Exception raised when censoring occurs during export.""" + + pass + + +class ExportFailedException(CaliperExportError): + """Exception raised when export fails.""" + + pass + + # --------------------------------------------------------------------------- # Run naming helpers # --------------------------------------------------------------------------- @@ -150,6 +171,8 @@ def resolve_run_names( def run_from_orchestration_config( caliper_cfg: dict[str, Any] | None, + disable_censoring: bool = False, + disable_file_export: bool = False, ) -> int: """ Run Caliper file export from orchestration config. @@ -250,6 +273,8 @@ def run_from_orchestration_config( verbose=export_cfg.verbose, status_yaml_path=status_yaml, upload_workers=export_cfg.upload_workers, + disable_censoring=disable_censoring, + disable_file_export=disable_file_export, ) else: effective_name = ( @@ -265,21 +290,46 @@ def run_from_orchestration_config( if mlflow_config_data is not None: mlflow_kwargs["mlflow_config_data"] = mlflow_config_data - ret = run_artifacts_export( - from_path=from_path, - status_yaml_path=status_yaml, - dry_run=export_cfg.dry_run, - verbose=export_cfg.verbose, - upload_workers=export_cfg.upload_workers, - backend=backends, - **mlflow_kwargs, - ) + # Apply censoring if enabled (in-place modification) + censoring_occurred = orchestration_apply_censoring(from_path, export_cfg, disable_censoring) + + if disable_file_export: + # Create mock status for notifications + mock_status = { + "success": True, + "final_status": "success", + "backends": {"mlflow": {"success": True, "run_id": "mock-disabled-export-id"}}, + "duration": "0 seconds (export disabled)", + "censoring_occurred": censoring_occurred, + } + # Write mock status to status file + with open(status_yaml, "w") as f: + yaml.dump(mock_status, f, indent=4) + else: + ret = run_artifacts_export( + from_path=from_path, + status_yaml_path=status_yaml, + dry_run=export_cfg.dry_run, + verbose=export_cfg.verbose, + upload_workers=export_cfg.upload_workers, + backend=backends, + **mlflow_kwargs, + ) + if ret != 0: + raise ExportFailedException(f"Artifacts export failed (ret code = {ret})") - if ret != 0: - raise RuntimeError(f"Caliper export failed (ret code = {ret})") + # Check for censoring in single-run export + if censoring_occurred: + raise CensoringOccurredException("Files were censored during export") with open(status_yaml) as f: - return yaml.safe_load(f.read()) + status = yaml.safe_load(f.read()) + + # Add censoring information to status + if len(run_dirs) == 1: + status["censoring_occurred"] = censoring_occurred + + return status @requires( @@ -473,7 +523,7 @@ def build_mlflow_run_url( assert_tracking_uri_has_no_userinfo(tracking_uri) qs = f"?workspace={quote(workspace, safe='')}" if workspace else "" - return f"{tracking_uri}{qs}#/experiments/{experiment_id}/runs/{run_id}/artifacts" + return f"{tracking_uri}/{qs}#/experiments/{experiment_id}/runs/{run_id}/artifacts" def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index ed033b268..4d3fe734f 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -12,8 +12,8 @@ from typing import Any import yaml -from projects.caliper.orchestration.censoring import censor_text +from projects.caliper.orchestration.censoring import censor_text from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.library import ci as ci_lib from projects.core.library import config, env @@ -674,12 +674,12 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: status_emoji = ( "✅" if status == "success" else "❌" if status == "failed" else "⚪" ) - step_statuses.append(f"{step_name}:{status_emoji}") + step_statuses.append(f"{status_emoji} {step_name}") # Format overall line overall_emoji = "✅" if overall_success else "❌" if step_statuses: - steps_str = " " + " ".join(step_statuses) + steps_str = " " + " • ".join(step_statuses) else: steps_str = f" {final_status}" From 87b0acc16ba596419f97ab33907f7999ae7477ec Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 11:33:09 +0200 Subject: [PATCH 0796/1072] [caliper] orchestration: censoring: add text censoring --- projects/caliper/orchestration/censoring.py | 122 +++++++--- projects/caliper/orchestration/export.py | 216 ++++++++++++++++-- projects/core/library/export.py | 37 +-- projects/core/library/export_notifications.py | 18 +- 4 files changed, 295 insertions(+), 98 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 032c842f9..6e2817dc4 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -20,23 +20,6 @@ logger = logging.getLogger(__name__) -# Vault content types to automatically discover for censoring -VAULT_CONTENT_TYPES = [ - "token", - "password", - "secret", - "key", - "credential", - "tracking_uri", - "username", - "api_key", - "api-key", - "auth", - "authentication", - "bearer", - "oauth", -] - def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, str]]: """ @@ -76,35 +59,101 @@ def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, s continue content_path = vault_manager.get_vault_content_path(vault_name, content_name) - if content_path and content_path.exists(): - try: - # Read vault content (assume it's text) - content_text = content_path.read_text( - encoding="utf-8", errors="ignore" - ).strip() - if content_text: - vault_secrets.add(content_text) - secret_mapping[content_text] = f"{vault_name}/{content_name}" - secrets_discovered += 1 - if verbose: - logger.info( - f"Discovered secret from vault {vault_name}/{content_name}" - ) - except Exception as e: + if not (content_path and content_path.exists()): + logger.warning(f"Invalid vault found: {vault_name} {content_name} (missing)") + continue + + try: + # Read vault content (assume it's text) + content_text = content_path.read_text(encoding="utf-8", errors="ignore").strip() + if not content_text: + logger.warning(f"Invalid vault found: {vault_name} {content_name} (empty)") + else: + vault_secrets.add(content_text) + secret_mapping[content_text] = f"{vault_name}/{content_name}" + secrets_discovered += 1 if verbose: - logger.warning( - f"Failed to read vault content {vault_name}/{content_name}: {e}" - ) + logger.info(f"Discovered secret from vault {vault_name}/{content_name}") + except Exception as e: + logger.warning(f"Failed to read vault content {vault_name}/{content_name}: {e}") if verbose: logger.info(f"Discovered {secrets_discovered} vault secrets for censoring") except Exception as e: - logger.warning(f"Failed to discover vault secrets: {e}") + logger.exception(f"Failed to discover vault secrets: {e}") + raise return vault_secrets, secret_mapping +def censor_text(text: str, verbose: bool = False) -> str: + """ + Censor sensitive content in text using caliper's censoring rules. + + Args: + text: The text to censor + verbose: Enable verbose logging + + Returns: + Censored text with sensitive content replaced + """ + if not text: + return text + + censored_text = text + replacements_made = 0 + + try: + # Discover vault secrets for censoring + vault_secrets, secret_mapping = discover_vault_secrets(verbose=verbose) + # /!\ secret_mapping contains the secret values. Process with extra care. + + # Replace vault secrets first (more specific) + for secret in vault_secrets: + if secret and secret.strip() and secret.strip() in censored_text: + censored_text = censored_text.replace(secret.strip(), "[REDACTED-VAULT]") + replacements_made += 1 + if verbose: + vault_identifier = secret_mapping.get(secret.strip(), "unknown vault") + logger.info(f"Censored vault secret from {vault_identifier} in text") + + # Import keyword patterns for content censoring + from projects.caliper.engine.file_export.censoring_rules import ( + COMPILED_KEYWORD_PATTERNS, + KEYWORD_PATTERNS, + ) + + # Apply keyword pattern censoring + for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + matches = list(pattern.finditer(censored_text)) + if matches: + # Replace all matched spans with redacted text, preserving other content + # Process patterns in reverse order by position to maintain string indices + for match in reversed(matches): + censored_text = ( + censored_text[: match.start()] + "[REDACTED]" + censored_text[match.end() :] + ) + replacements_made += 1 + + if verbose: + logger.info( + f"Censored {len(matches)} instances of pattern '{KEYWORD_PATTERNS[i]}' in text" + ) + + if replacements_made > 0: + logger.info(f"Censored {replacements_made} sensitive items from text") + elif verbose: + logger.info("No sensitive content found in text") + + except Exception as e: + logger.warning(f"Error during text censoring: {e}") + # On error, add a warning to the text + censored_text += "\n\n> ⚠️ **Note**: Automatic content censoring encountered an error. Please review this content manually for sensitive information." + + return censored_text + + def orchestration_apply_censoring( from_path: Path, export_cfg: CaliperOrchestrationExportConfig, @@ -128,6 +177,7 @@ def orchestration_apply_censoring( # Discover vault secrets vault_secrets, secret_mapping = discover_vault_secrets(verbose=export_cfg.verbose) + # /!\ secret_mapping contains the secret values. Process with extra care. # Collect all artifact files all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 5b119d4d6..8d9342539 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -22,9 +22,11 @@ from projects.caliper.engine.file_export.artifacts_export_run import ( discover_run_dirs, run_artifacts_export, - run_multi_run_artifacts_export, ) -from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml +from projects.caliper.engine.file_export.mlflow_config import ( + load_mlflow_config_yaml, + project_metadata_fields, +) from projects.caliper.orchestration.censoring import ( orchestration_apply_censoring, ) @@ -44,12 +46,6 @@ class CaliperExportError(Exception): pass -class CensoringOccurredException(CaliperExportError): - """Exception raised when censoring occurs during export.""" - - pass - - class ExportFailedException(CaliperExportError): """Exception raised when export fails.""" @@ -258,21 +254,16 @@ def run_from_orchestration_config( logger.info( "dry-run: would export %d run dirs from %s (skipping)", len(run_dirs), from_path ) - ret = 0 else: - ret = run_multi_run_artifacts_export( + _run_multi_run_export( + export_cfg=export_cfg, from_path=from_path, - run_dirs=run_dirs, - backend=backends, - mlflow_experiment=export_cfg.mlflow_experiment, - mlflow_run_name=naming.get("parent_run_name"), + status_yaml=status_yaml, mlflow_secrets_path=mlflow_secrets_path, mlflow_config_data=mlflow_config_data, - mlflow_run_id=mlflow_run_id, + run_dirs=run_dirs, + resolved_parent_name=naming.get("parent_run_name"), child_run_names=naming.get("child_run_names") or {}, - verbose=export_cfg.verbose, - status_yaml_path=status_yaml, - upload_workers=export_cfg.upload_workers, disable_censoring=disable_censoring, disable_file_export=disable_file_export, ) @@ -298,7 +289,9 @@ def run_from_orchestration_config( mock_status = { "success": True, "final_status": "success", - "backends": {"mlflow": {"success": True, "run_id": "mock-disabled-export-id"}}, + "caliper_artifacts_export": { + "backends": {"mlflow": {"success": True, "run_id": "mock-disabled-export-id"}}, + }, "duration": "0 seconds (export disabled)", "censoring_occurred": censoring_occurred, } @@ -318,10 +311,6 @@ def run_from_orchestration_config( if ret != 0: raise ExportFailedException(f"Artifacts export failed (ret code = {ret})") - # Check for censoring in single-run export - if censoring_occurred: - raise CensoringOccurredException("Files were censored during export") - with open(status_yaml) as f: status = yaml.safe_load(f.read()) @@ -549,3 +538,184 @@ def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: logger.warning("Failed to read test labels %s: %s", labels_file, e) return None + + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" + + +def _discover_run_dirs(from_path: Path) -> list[Path]: + """Auto-detect test run directories via ``__test_labels__.yaml`` markers.""" + run_dirs: list[Path] = [] + for marker in sorted(from_path.rglob(TEST_LABELS_MARKER)): + if marker.is_file(): + run_dirs.append(marker.parent) + + if run_dirs: + logger.info( + "Auto-detected %d test run director%s via %s", + len(run_dirs), + "y" if len(run_dirs) == 1 else "ies", + TEST_LABELS_MARKER, + ) + return run_dirs + + +def _run_multi_run_export( + *, + export_cfg: CaliperOrchestrationExportConfig, + from_path: Path, + status_yaml: Path, + mlflow_secrets_path: Path, + mlflow_config_data: dict[str, Any] | None, + run_dirs: list[Path], + resolved_parent_name: str | None = None, + child_run_names: dict[Path, str] | None = None, + disable_censoring: bool = False, + disable_file_export: bool = False, +) -> None: + """Export as parent + nested child MLflow runs. + + Raises: + ExportFailedException: If the export fails + """ + import sys + import traceback + + import click + + from projects.caliper.engine.file_export import mlflow_backend + from projects.caliper.engine.file_export.artifacts_export_run import ( + merge_mlflow_files_with_cli, + write_artifacts_status_yaml, + ) + from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + project_secrets_fields, + validate_mlflow_secrets, + ) + from projects.caliper.engine.model import FileExportBackendResult + + logger.info("Multi-run export: %d test run(s) detected", len(run_dirs)) + + all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] + + # Apply censoring for multi-run export if enabled + censoring_occurred = orchestration_apply_censoring(from_path, export_cfg, disable_censoring) + + # Update artifact paths - use original paths since we modified in-place + all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] + + secrets_data = None + if mlflow_secrets_path is not None: + secrets_data = load_mlflow_secrets_yaml(mlflow_secrets_path) + validate_mlflow_secrets(secrets_data) + + merged_ml = merge_mlflow_files_with_cli( + None, + secrets_data=secrets_data, + config_data=mlflow_config_data, + cli_tracking_uri=None, + cli_experiment=export_cfg.mlflow_experiment, + cli_run_id=None, + cli_run_name=export_cfg.mlflow_run_name, + ) + + secret_part = project_secrets_fields(merged_ml) + mlflow_connection = secret_part if secret_part else None + + tracking_uri = merged_ml.get("tracking_uri") + experiment = merged_ml.get("experiment") + run_name = resolved_parent_name or merged_ml.get("run_name") + workspace = merged_ml.get("workspace") + if not workspace: + raise ValueError("The export workspace must be specified") + + meta = project_metadata_fields(merged_ml) + run_metadata = meta if meta else None + + insecure_tls = bool(mlflow_connection and mlflow_connection.get("insecure_tls")) + + if export_cfg.verbose: + click.echo("caliper multi-run export (verbose)", err=True) + click.echo(f" Source: {from_path}", err=True) + click.echo(f" Total artifact files: {len(all_artifact_paths)}", err=True) + click.echo(f" Run directories: {len(run_dirs)}", err=True) + click.echo(f" Workspace: {workspace}", err=True) + for rd in run_dirs: + click.echo(f" - {rd.name}", err=True) + click.echo("", err=True) + + try: + if disable_file_export: + # Create mock results for notifications + detail = "" + ml_meta = { + "run_id": "mock-multi-run-disabled-export", + "experiment_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/disabled", + "run_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/disabled/runs/mock-multi-run-disabled-export", + "tracking_uri": "http://DRY_RUN_MLFLOW_FAKE_URL", + } + results = [ + FileExportBackendResult( + backend="mlflow", + status="success", + detail=detail, + metadata=ml_meta, + ) + ] + else: + detail, ml_meta = mlflow_backend.log_multi_run_artifacts( + all_artifact_paths=all_artifact_paths, + artifact_root=from_path, + run_dirs=run_dirs, + metrics_file=METRICS_FILE, + parameters_file=PARAMETERS_FILE, + tracking_uri=tracking_uri, + experiment=experiment, + parent_run_name=run_name, + insecure_tls=insecure_tls, + connection=mlflow_connection, + verbose=export_cfg.verbose, + upload_workers=export_cfg.upload_workers, + run_metadata=run_metadata, + workspace=workspace, + child_run_names=child_run_names or None, + ) + results = [ + FileExportBackendResult( + backend="mlflow", + status="success", + detail=detail, + metadata=ml_meta, + ) + ] + except Exception as e: + traceback.print_exception(e, file=sys.stderr) + click.echo(f"multi-run export failed: {e}", err=True) + results = [FileExportBackendResult(backend="mlflow", status="failure", detail=str(e))] + + if not disable_file_export: + for r in results: + click.echo(f"{r.backend}: {r.status} {r.detail}") + + if status_yaml is not None: + try: + write_artifacts_status_yaml(status_yaml, results) + + # Add censoring information to the status file + with open(status_yaml) as f: + status_data = yaml.safe_load(f) + status_data["censoring_occurred"] = censoring_occurred + with open(status_yaml, "w") as f: + yaml.dump(status_data, f, indent=4) + + if not disable_file_export: + click.echo(f"Wrote status YAML to {status_yaml}") + except OSError as e: + click.echo(f"Failed to write status YAML ({status_yaml}): {e}", err=True) + raise ExportFailedException(f"Failed to write status YAML: {e}") from None + + if any(r.status == "failure" for r in results): + raise ExportFailedException("MLflow backend export failed") diff --git a/projects/core/library/export.py b/projects/core/library/export.py index eb4cb1c79..2f601574b 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -17,7 +17,6 @@ import yaml from projects.caliper.orchestration.export import ( - CensoringOccurredException, ExportFailedException, run_from_orchestration_config, ) @@ -181,35 +180,15 @@ def run_caliper_orchestration_export( ) # Initialize vaults needed for export operations - logger.info("Checking vaults for export operations") try: + from projects.core.library import vault + # Get export-specific vaults (MLflow, S3, notifications) export_vaults = caliper_export_list_vaults() - logger.info(f"Export vaults needed: {len(export_vaults)} - {export_vaults}") - # Initialize vaults if any are needed if export_vaults: - from projects.core.library import vault - - # Check if vault manager is already initialized - try: - vault.get_vault_manager() - logger.info( - f"Vault manager already initialized, checking {len(export_vaults)} export vaults" - ) - manager_already_initialized = True - except RuntimeError: - logger.info(f"Initializing vault manager with {len(export_vaults)} export vaults") - manager_already_initialized = False - vault.init(vaults=export_vaults) - - if manager_already_initialized: - logger.info(f"Export vault check completed for {len(export_vaults)} vaults") - else: - logger.info( - f"Successfully initialized vault manager with {len(export_vaults)} vaults for export" - ) + logger.info(f"Initialized vault manager with {len(export_vaults)} vaults for export") else: logger.info("No vaults needed for export operation") @@ -352,16 +331,6 @@ def caliper_export_entrypoint( else: logger.info("Skipping fjob status update due to --disable-file-export flag") - except CensoringOccurredException as e: - logger.info(f"Export completed with censoring: {e}") - # Create success status with censoring flag for notification - status = { - "success": True, - "final_status": "success_with_censoring", - "censoring_occurred": True, - "message": str(e), - "backends": {}, - } except ExportFailedException as e: logger.exception(f"Export failed: {e}") export_failed = True diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 4d3fe734f..0be57b464 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -124,8 +124,12 @@ def send_notification( from projects.core.notifications.provider import NotificationContext artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None + # Censor status fields before creating NotificationContext + censored_status = yaml.safe_load( + _censor_notification_text(yaml.dump(status), verbose=dry_run) + ) context = NotificationContext( - status=status, + status=censored_status, finish_reason=str(finish_reason), project_name=project or "unknown", pr_number=os.environ.get("PULL_NUMBER"), @@ -338,12 +342,14 @@ def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | No log_counts = _count_log_messages(step_dir) log_summary = _format_log_summary(log_counts) - mlflow_log_url = ( - f"{mlflow_run_url}/artifacts/{step_name}/run.log" if mlflow_run_url else "#" - ) + # Create step title - linked if MLflow URL available, plain-text otherwise if mlflow_run_url: + mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_name) step_title = f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}){log_summary}" - step_status.append(step_title) + else: + step_title = f"#### {exit_status_emoji} {step_name}{log_summary}" + + step_status.append(step_title) step_status.extend(_process_notification_files(step_dir)) @@ -604,6 +610,8 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N if message: message = f": `{message}`" + else: + message = "" # Format status if success: From a8c2d9274d13bba7aaddc01eac842edb26c52976 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 12:07:23 +0200 Subject: [PATCH 0797/1072] [skeleton] orchestration: test_skeleton: reduce the log level --- projects/skeleton/orchestration/test_skeleton.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/skeleton/orchestration/test_skeleton.py b/projects/skeleton/orchestration/test_skeleton.py index 6523fb9a7..a29a4e42b 100644 --- a/projects/skeleton/orchestration/test_skeleton.py +++ b/projects/skeleton/orchestration/test_skeleton.py @@ -97,15 +97,15 @@ def do_test(): logger.info("=== Skeleton Project Test Phase ===") if config.project.get_config("skeleton.deep_testing"): - logger.warning("Running the (fake) deep testing ...") + logger.info("Running the (fake) deep testing ...") else: - logger.warning("Running the (fake) light testing ...") + logger.info("Running the (fake) light testing ...") client_id = vault.get_vault_content_path("psap-forge-notifications", "topsail-bot.clientid") if not client_id: - logger.warning("`client_id` secret not available.") + logger.error("`client_id` secret not available.") else: - logger.warning(f"`client_id` secret available. Size: {client_id.stat().st_size}b") + logger.info(f"`client_id` secret available. Size: {client_id.stat().st_size}b") del client_id skeleton_config = config.project.get_config("skeleton", print=False) @@ -125,7 +125,7 @@ def do_test(): seed_skeleton_caliper_artifacts() if not config.project.get_config("skeleton.collect_cluster_info"): - logger.warning("⚠️ Cluster information gathering not enabled. Returning early.") + logger.info("⚠️ Cluster information gathering not enabled. Returning early.") return 0 # Demonstrate calling a toolbox from orchestration From c9d38a6beebfb4386e6009f7c232349e30e67708 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 29 Jul 2026 08:34:31 +0200 Subject: [PATCH 0798/1072] vaults: psap-forge-mlflow-export: mark the username as not sensible --- vaults/psap-forge-mlflow-export.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vaults/psap-forge-mlflow-export.yaml b/vaults/psap-forge-mlflow-export.yaml index 1538e8d2f..c4c2425e1 100644 --- a/vaults/psap-forge-mlflow-export.yaml +++ b/vaults/psap-forge-mlflow-export.yaml @@ -17,8 +17,9 @@ content: the URI of the MLFlow endpoint, so that the CI engine can censor it in the logs username: + sensible: false description: | - the mlflow username, so that the CI engine can censor it in the logs + the mlflow username password: description: | From 69ddb425d8eb437e91f79e26d10e1cc192277c4d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 30 Jul 2026 13:26:25 +0200 Subject: [PATCH 0799/1072] vaults: psap-forge-notifications: mark the GH bot ID as non sensible --- vaults/psap-forge-notifications.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 12ae01cc8..73eb1ce5b 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -7,8 +7,10 @@ content: description: | Private key to send github notifications topsail-bot.clientid: + sensible: false description: | - Token to identify the TOPSAIL-bot client + Public ID of the TOPSAIL-bot client + topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot From 47ffb5087c56952703d1c74bd15fe8197016102b Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Wed, 26 Aug 2026 08:26:18 +0000 Subject: [PATCH 0800/1072] fix: address review feedback and add tests for caliper censoring Critical fixes: - Fix sanitized files being misclassified as censored in CLI command, preventing data loss where redacted-in-place files were deleted - Fix multi-run dry-run falling through to read non-existent status.yaml by returning early with a dry-run status dict Major fixes: - Fix overlapping regex spans corrupting redacted output by merging overlapping spans before applying replacements in reverse order - Remove dead first rglob scan in multi-run export that was immediately overwritten Minor fixes: - Remove redundant bearer token pattern (duplicate with IGNORECASE) - Remove unused find_sensitive_content_in_text function Tests: - Add 65 comprehensive tests covering pattern matching, sanitization, overlapping span merging, dry-run mode, file classification, vault secret replacement, and end-to-end directory censoring Co-Authored-By: Claude Opus 4.6 --- projects/caliper/cli/commands.py | 72 ++- .../caliper/engine/file_export/censoring.py | 43 +- .../engine/file_export/censoring_rules.py | 22 +- projects/caliper/orchestration/export.py | 10 +- projects/caliper/tests/test_censoring.py | 558 ++++++++++++++++++ 5 files changed, 646 insertions(+), 59 deletions(-) create mode 100644 projects/caliper/tests/test_censoring.py diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 90d3cc3f8..cba9aedf4 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -1412,22 +1412,31 @@ def artifacts_censor( all_artifact_paths, censoring_enabled=True, verbose=verbose, dry_run=dry_run ) - # Separate censored and clean files - censored_files = [r.file_path for r in censoring_results if r.censored] + # Separate results: excluded (fully censored), sanitized (redacted in-place), and clean + excluded_files = [r.file_path for r in censoring_results if r.censored and not r.sanitized] + sanitized_files = [r.file_path for r in censoring_results if r.sanitized] clean_files = [r.file_path for r in censoring_results if not r.censored] # Print summary click.echo("\nCensoring Summary:") click.echo(f" Total files scanned: {len(all_artifact_paths)}") click.echo(f" Clean files: {len(clean_files)}") - click.echo(f" Censored files: {len(censored_files)}") - - if censored_files and verbose: - click.echo("\nCensored files:") - for result in censoring_results: - if result.censored: - rel_path = result.file_path.relative_to(from_path) - click.echo(f" - {rel_path}: {result.reason}") + click.echo(f" Sanitized files: {len(sanitized_files)}") + click.echo(f" Excluded files: {len(excluded_files)}") + + if (excluded_files or sanitized_files) and verbose: + if sanitized_files: + click.echo("\nSanitized files (redacted in-place, kept):") + for result in censoring_results: + if result.sanitized: + rel_path = result.file_path.relative_to(from_path) + click.echo(f" - {rel_path}: {result.reason}") + if excluded_files: + click.echo("\nExcluded files (to be removed):") + for result in censoring_results: + if result.censored and not result.sanitized: + rel_path = result.file_path.relative_to(from_path) + click.echo(f" - {rel_path}: {result.reason}") # Generate report if requested if report_path: @@ -1437,14 +1446,23 @@ def artifacts_censor( "output_directory": str(output_path) if output_path else None, "total_files": len(all_artifact_paths), "clean_files": len(clean_files), - "censored_files": len(censored_files), - "censored_details": [ + "sanitized_files": len(sanitized_files), + "excluded_files": len(excluded_files), + "sanitized_details": [ + { + "file": str(r.file_path.relative_to(from_path)), + "reason": r.reason, + } + for r in censoring_results + if r.sanitized + ], + "excluded_details": [ { "file": str(r.file_path.relative_to(from_path)), "reason": r.reason, } for r in censoring_results - if r.censored + if r.censored and not r.sanitized ], } @@ -1465,38 +1483,40 @@ def artifacts_censor( if dry_run: click.echo("\nDry run: No files were modified") if output_path: - click.echo(f"Would copy {len(clean_files)} clean files to {output_path}") + kept_count = len(clean_files) + len(sanitized_files) + click.echo(f"Would copy {kept_count} files to {output_path}") else: - click.echo(f"Would remove {len(censored_files)} censored files from {from_path}") + click.echo(f"Would remove {len(excluded_files)} excluded files from {from_path}") elif output_path: - # Copy clean files to output directory + # Copy clean and sanitized files to output directory (exclude only truly excluded files) + kept_files = clean_files + sanitized_files try: output_path.mkdir(parents=True, exist_ok=True) - for clean_file in clean_files: - rel_path = clean_file.relative_to(from_path) + for kept_file in kept_files: + rel_path = kept_file.relative_to(from_path) dest_path = output_path / rel_path dest_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(clean_file, dest_path) + shutil.copy2(kept_file, dest_path) - click.echo(f"\nCopied {len(clean_files)} clean files to {output_path}") + click.echo(f"\nCopied {len(kept_files)} files to {output_path}") except Exception as e: click.echo(f"Error copying files: {e}", err=True) sys.exit(1) else: - # Remove censored files in-place - if censored_files: + # Remove only excluded files in-place (sanitized files are already redacted and kept) + if excluded_files: try: - for censored_file in censored_files: - censored_file.unlink() + for excluded_file in excluded_files: + excluded_file.unlink() if verbose: - rel_path = censored_file.relative_to(from_path) + rel_path = excluded_file.relative_to(from_path) click.echo(f" Removed: {rel_path}") - click.echo(f"\nRemoved {len(censored_files)} censored files from {from_path}") + click.echo(f"\nRemoved {len(excluded_files)} excluded files from {from_path}") except Exception as e: click.echo(f"Error removing files: {e}", err=True) diff --git a/projects/caliper/engine/file_export/censoring.py b/projects/caliper/engine/file_export/censoring.py index b8218b764..6aead82db 100644 --- a/projects/caliper/engine/file_export/censoring.py +++ b/projects/caliper/engine/file_export/censoring.py @@ -21,6 +21,32 @@ logger = logging.getLogger(__name__) +def _merge_overlapping_spans(spans: list[tuple[int, int]]) -> list[tuple[int, int]]: + """Merge overlapping or adjacent (start, end) spans into non-overlapping spans. + + Args: + spans: List of (start, end) tuples representing character ranges. + + Returns: + Sorted list of merged (start, end) tuples with no overlaps. + """ + if not spans: + return [] + + sorted_spans = sorted(spans) + merged = [sorted_spans[0]] + + for start, end in sorted_spans[1:]: + prev_start, prev_end = merged[-1] + if start <= prev_end: + # Overlapping or adjacent — extend the previous span + merged[-1] = (prev_start, max(prev_end, end)) + else: + merged.append((start, end)) + + return merged + + @dataclass class CensoringResult: """Result of censoring operation on a single file.""" @@ -153,18 +179,17 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: matched_patterns.add(i) if keyword_detected: - # Replace all matched spans with redacted text, preserving other content - # Process patterns in reverse order by position to maintain string indices - all_matches = [] - for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + # Collect all match spans from all patterns + all_spans = [] + for pattern in COMPILED_KEYWORD_PATTERNS: for match in pattern.finditer(content): - all_matches.append((match.start(), match.end(), i)) + all_spans.append((match.start(), match.end())) - # Sort by start position in reverse order - all_matches.sort(reverse=True) + # Merge overlapping spans to avoid corrupting replacements + merged_spans = _merge_overlapping_spans(all_spans) - # Replace each match with [REDACTED] - for start, end, _pattern_idx in all_matches: + # Replace merged spans in reverse order to preserve string indices + for start, end in reversed(merged_spans): content = content[:start] + "[REDACTED]" + content[end:] # Add reasons for all matched patterns diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index c2b2140a8..1c56b0bf7 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -24,9 +24,8 @@ r"secret[_-]?token\s*[:=]\s*\S+", r"access[_-]?token\s*[:=]\s*\S+", r"refresh[_-]?token\s*[:=]\s*\S+", - # Bearer tokens + # Bearer tokens (compiled with IGNORECASE, so one pattern covers both cases) r"Bearer\s+[A-Za-z0-9+/=]+", - r"bearer\s+[A-Za-z0-9+/=]+", # Specific service API keys r"sk-[a-zA-Z0-9]{32,}", # OpenAI API keys r"ghp_[a-zA-Z0-9]{36}", # GitHub personal access tokens @@ -106,22 +105,3 @@ def matches_sensitive_filename(filename: str) -> bool: return True return False - - -def find_sensitive_content_in_text(content: str) -> list[str]: - """ - Find sensitive content patterns in text. - - Args: - content: Text content to scan - - Returns: - list[str]: List of pattern descriptions that matched - """ - matches = [] - for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): - if pattern.search(content): - # Return the original pattern string for logging - matches.append(KEYWORD_PATTERNS[i]) - - return matches diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 8d9342539..696ed8f02 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -254,6 +254,12 @@ def run_from_orchestration_config( logger.info( "dry-run: would export %d run dirs from %s (skipping)", len(run_dirs), from_path ) + return { + "success": True, + "final_status": "dry-run", + "dry_run": True, + "run_dirs": len(run_dirs), + } else: _run_multi_run_export( export_cfg=export_cfg, @@ -599,12 +605,10 @@ def _run_multi_run_export( logger.info("Multi-run export: %d test run(s) detected", len(run_dirs)) - all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] - # Apply censoring for multi-run export if enabled censoring_occurred = orchestration_apply_censoring(from_path, export_cfg, disable_censoring) - # Update artifact paths - use original paths since we modified in-place + # Collect artifact paths after censoring (files may have been modified in-place) all_artifact_paths = [p for p in from_path.rglob("*") if p.is_file()] secrets_data = None diff --git a/projects/caliper/tests/test_censoring.py b/projects/caliper/tests/test_censoring.py new file mode 100644 index 000000000..2708df689 --- /dev/null +++ b/projects/caliper/tests/test_censoring.py @@ -0,0 +1,558 @@ +""" +Tests for the caliper artifact censoring module. + +Covers: +- Keyword pattern matching and redaction +- Filename-based sensitive file detection +- Content sanitization with in-place replacement +- Overlapping regex span merging +- Dry-run mode behavior +- Separation of censored vs sanitized files +- File exclusion vs sanitization classification +- End-to-end censoring with temporary directories +- Bearer token pattern deduplication +- Vault secret replacement +""" + +from __future__ import annotations + +from pathlib import Path + +from projects.caliper.engine.file_export.censoring import ( + ArtifactCensor, + CensoringResult, + _merge_overlapping_spans, + apply_censoring_to_artifacts, +) +from projects.caliper.engine.file_export.censoring_rules import ( + COMPILED_KEYWORD_PATTERNS, + KEYWORD_PATTERNS, + matches_sensitive_filename, +) + +# --------------------------------------------------------------------------- +# _merge_overlapping_spans +# --------------------------------------------------------------------------- + + +class TestMergeOverlappingSpans: + def test_empty_list(self): + assert _merge_overlapping_spans([]) == [] + + def test_single_span(self): + assert _merge_overlapping_spans([(0, 5)]) == [(0, 5)] + + def test_non_overlapping_spans(self): + spans = [(0, 5), (10, 15), (20, 25)] + assert _merge_overlapping_spans(spans) == [(0, 5), (10, 15), (20, 25)] + + def test_overlapping_spans_are_merged(self): + spans = [(0, 10), (5, 15)] + assert _merge_overlapping_spans(spans) == [(0, 15)] + + def test_adjacent_spans_are_merged(self): + spans = [(0, 5), (5, 10)] + assert _merge_overlapping_spans(spans) == [(0, 10)] + + def test_nested_spans_are_merged(self): + spans = [(0, 20), (5, 10)] + assert _merge_overlapping_spans(spans) == [(0, 20)] + + def test_multiple_overlapping_groups(self): + spans = [(0, 5), (3, 8), (10, 15), (12, 18)] + assert _merge_overlapping_spans(spans) == [(0, 8), (10, 18)] + + def test_unsorted_input_is_handled(self): + spans = [(10, 15), (0, 5), (3, 8)] + assert _merge_overlapping_spans(spans) == [(0, 8), (10, 15)] + + def test_fully_contained_spans(self): + spans = [(0, 100), (10, 20), (30, 40), (50, 60)] + assert _merge_overlapping_spans(spans) == [(0, 100)] + + +# --------------------------------------------------------------------------- +# matches_sensitive_filename +# --------------------------------------------------------------------------- + + +class TestMatchesSensitiveFilename: + def test_pem_file(self): + assert matches_sensitive_filename("server.pem") + + def test_key_file(self): + assert matches_sensitive_filename("private.key") + + def test_p12_file(self): + assert matches_sensitive_filename("cert.p12") + + def test_pfx_file(self): + assert matches_sensitive_filename("cert.pfx") + + def test_env_file(self): + assert matches_sensitive_filename(".env") + + def test_env_with_suffix(self): + assert matches_sensitive_filename(".env.production") + + def test_secret_in_name(self): + assert matches_sensitive_filename("my_secret_config.yaml") + + def test_credential_in_name(self): + assert matches_sensitive_filename("credential_store.json") + + def test_password_in_name(self): + assert matches_sensitive_filename("password_file.txt") + + def test_ssh_key(self): + assert matches_sensitive_filename("id_rsa") + + def test_ecdsa_key(self): + assert matches_sensitive_filename("id_ecdsa") + + def test_ed25519_key(self): + assert matches_sensitive_filename("id_ed25519") + + def test_normal_text_file(self): + assert not matches_sensitive_filename("readme.txt") + + def test_normal_yaml_file(self): + assert not matches_sensitive_filename("config.yaml") + + def test_normal_log_file(self): + assert not matches_sensitive_filename("output.log") + + def test_path_with_secret_in_basename(self): + assert matches_sensitive_filename("/some/path/secret_config.yaml") + + def test_case_insensitive_pem(self): + assert matches_sensitive_filename("CERT.PEM") + + def test_case_insensitive_secret(self): + assert matches_sensitive_filename("MY_SECRET.txt") + + +# --------------------------------------------------------------------------- +# Keyword pattern matching +# --------------------------------------------------------------------------- + + +class TestKeywordPatterns: + """Test that compiled keyword patterns detect sensitive content.""" + + def _matches_any_pattern(self, text: str) -> bool: + return any(p.search(text) for p in COMPILED_KEYWORD_PATTERNS) + + def test_password_pattern(self): + assert self._matches_any_pattern("password=hunter2") + + def test_password_colon(self): + assert self._matches_any_pattern("password: my_password") + + def test_api_key_pattern(self): + assert self._matches_any_pattern("api_key=abc123xyz") + + def test_api_secret_pattern(self): + assert self._matches_any_pattern("api-secret=supersecret") + + def test_token_pattern(self): + assert self._matches_any_pattern("token=eyJhbGciOiJIUzI1NiJ9") + + def test_bearer_token(self): + assert self._matches_any_pattern("Bearer eyJhbGciOiJIUzI1NiJ9") + + def test_bearer_case_insensitive(self): + # Since patterns are compiled with IGNORECASE, lowercase should match too + assert self._matches_any_pattern("bearer eyJhbGciOiJIUzI1NiJ9") + + def test_openai_key(self): + assert self._matches_any_pattern("sk-abcdefghijklmnopqrstuvwxyz123456") + + def test_github_token(self): + assert self._matches_any_pattern("ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij") + + def test_aws_access_key(self): + assert self._matches_any_pattern("AKIAIOSFODNN7EXAMPLE") + + def test_mongodb_uri(self): + assert self._matches_any_pattern("mongodb://user:pass@host") + + def test_postgresql_uri(self): + assert self._matches_any_pattern("postgresql://admin:secret@db.example.com") + + def test_clean_text(self): + assert not self._matches_any_pattern("This is a normal log line with no secrets") + + def test_clean_json(self): + assert not self._matches_any_pattern('{"name": "test", "value": 42}') + + def test_no_duplicate_bearer_pattern(self): + # After removing the redundant lowercase bearer pattern, + # verify the single pattern still works for both cases + bearer_patterns = [p for p in KEYWORD_PATTERNS if "earer" in p] + assert len(bearer_patterns) == 1, ( + f"Expected exactly 1 bearer pattern, found {len(bearer_patterns)}: {bearer_patterns}" + ) + + +# --------------------------------------------------------------------------- +# CensoringResult +# --------------------------------------------------------------------------- + + +class TestCensoringResult: + def test_sanitized_str(self): + r = CensoringResult(Path("test.txt"), censored=True, reason="keyword", sanitized=True) + assert "SANITIZED" in str(r) + + def test_excluded_str(self): + r = CensoringResult(Path("test.txt"), censored=True, reason="keyword", sanitized=False) + assert "EXCLUDED" in str(r) + + def test_allowed_str(self): + r = CensoringResult(Path("test.txt"), censored=False, reason="clean", sanitized=False) + assert "ALLOWED" in str(r) + + +# --------------------------------------------------------------------------- +# ArtifactCensor._sanitize_file_content +# --------------------------------------------------------------------------- + + +class TestSanitizeFileContent: + def test_clean_file_passes(self, tmp_path): + f = tmp_path / "clean.txt" + f.write_text("Nothing sensitive here\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert not result.censored + assert not result.sanitized + + def test_password_is_redacted(self, tmp_path): + f = tmp_path / "config.txt" + f.write_text("password=hunter2\nother_line\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.censored + assert result.sanitized + content = f.read_text() + assert "hunter2" not in content + assert "[REDACTED]" in content + # Other content should be preserved + assert "other_line" in content + + def test_sensitive_filename_is_sanitized(self, tmp_path): + f = tmp_path / "secret_config.yaml" + f.write_text("important: data\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.censored + assert result.sanitized + content = f.read_text() + assert "Content censored by caliper" in content + assert "important: data" not in content + + def test_vault_secret_is_replaced(self, tmp_path): + f = tmp_path / "log.txt" + f.write_text("connecting with supersecrettoken123\n", encoding="utf-8") + + censor = ArtifactCensor( + vault_secrets={"supersecrettoken123"}, + secret_mapping={"supersecrettoken123": "vault/token"}, + ) + result = censor._sanitize_file_content(f) + + assert result.censored + assert result.sanitized + content = f.read_text() + assert "supersecrettoken123" not in content + assert "[REDACTED-VAULT]" in content + + def test_binary_file_skipped(self, tmp_path): + f = tmp_path / "image.dat" + f.write_bytes(bytes(range(256))) + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert not result.censored + assert not result.sanitized + + def test_dry_run_does_not_modify_file(self, tmp_path): + original_content = "password=hunter2\n" + f = tmp_path / "config.txt" + f.write_text(original_content, encoding="utf-8") + + censor = ArtifactCensor(dry_run=True) + result = censor._sanitize_file_content(f) + + assert result.censored + assert result.sanitized + # File should NOT be modified in dry run + assert f.read_text() == original_content + + def test_overlapping_patterns_produce_correct_output(self, tmp_path): + # token=xxx matches "token" pattern. "access_token=xxx" also matches. + # These could overlap if the text has "access_token=myvalue" + f = tmp_path / "config.txt" + f.write_text("access_token=myvalue123\nother content\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.sanitized + content = f.read_text() + # The redacted content should not be corrupted + assert "[REDACTED]" in content + assert "other content" in content + # Original secret should be gone + assert "myvalue123" not in content + + def test_multiple_secrets_on_same_line(self, tmp_path): + f = tmp_path / "multi.txt" + f.write_text("password=abc api_key=xyz\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.sanitized + content = f.read_text() + assert "abc" not in content + assert "xyz" not in content + assert content.count("[REDACTED]") >= 2 + + +# --------------------------------------------------------------------------- +# ArtifactCensor.censor_files +# --------------------------------------------------------------------------- + + +class TestCensorFiles: + def test_separates_clean_sanitized_excluded(self, tmp_path): + """Verify that clean, sanitized, and excluded files are handled correctly.""" + clean = tmp_path / "readme.txt" + clean.write_text("Hello world\n", encoding="utf-8") + + sensitive_content = tmp_path / "config.yaml" + sensitive_content.write_text("password=mysecret\n", encoding="utf-8") + + sensitive_name = tmp_path / "secret_data.yaml" + sensitive_name.write_text("key: value\n", encoding="utf-8") + + censor = ArtifactCensor() + processed, results = censor.censor_files([clean, sensitive_content, sensitive_name]) + + # All three should be in processed (clean + 2 sanitized) + assert len(processed) == 3 + assert clean in processed + assert sensitive_content in processed + assert sensitive_name in processed + + # Check result classifications + clean_results = [r for r in results if not r.censored] + sanitized_results = [r for r in results if r.sanitized] + + assert len(clean_results) == 1 + assert len(sanitized_results) == 2 + + def test_nonexistent_file_is_passed_through(self, tmp_path): + nonexistent = tmp_path / "does_not_exist.txt" + censor = ArtifactCensor() + processed, results = censor.censor_files([nonexistent]) + + assert nonexistent in processed + assert len(results) == 0 + + def test_verbose_logging(self, tmp_path): + f = tmp_path / "config.txt" + f.write_text("password=test\n", encoding="utf-8") + + censor = ArtifactCensor(verbose=True) + processed, results = censor.censor_files([f]) + + assert len(processed) == 1 + assert results[0].sanitized + + +# --------------------------------------------------------------------------- +# apply_censoring_to_artifacts +# --------------------------------------------------------------------------- + + +class TestApplyCensoringToArtifacts: + def test_censoring_disabled_returns_all(self, tmp_path): + f = tmp_path / "secret.pem" + f.write_text("private key data\n", encoding="utf-8") + + paths, results = apply_censoring_to_artifacts( + [f], censoring_enabled=False, verbose=True + ) + + assert f in paths + assert len(results) == 0 + + def test_end_to_end_with_mixed_files(self, tmp_path): + """End-to-end test with a directory containing various file types.""" + # Clean file + clean = tmp_path / "output.log" + clean.write_text("Test completed successfully\n", encoding="utf-8") + + # File with password + with_secret = tmp_path / "app.conf" + with_secret.write_text("database_password=abc123\nhost=localhost\n", encoding="utf-8") + + # File with sensitive filename + sensitive_name = tmp_path / "credentials.json" + sensitive_name.write_text('{"user": "admin"}\n', encoding="utf-8") + + # Binary file + binary = tmp_path / "image.png" + binary.write_bytes(b"\x89PNG\r\n\x1a\n" + bytes(100)) + + all_paths = [clean, with_secret, sensitive_name, binary] + processed, results = apply_censoring_to_artifacts(all_paths, censoring_enabled=True) + + # Clean and binary files pass through; secret-content and sensitive-name are sanitized + assert clean in processed + assert binary in processed + + # Verify content was sanitized + assert "abc123" not in with_secret.read_text() + assert "[REDACTED]" in with_secret.read_text() + + # Verify sensitive filename content was replaced + assert "Content censored by caliper" in sensitive_name.read_text() + + def test_dry_run_preserves_all_files(self, tmp_path): + f = tmp_path / "config.txt" + original = "password=secret123\n" + f.write_text(original, encoding="utf-8") + + processed, results = apply_censoring_to_artifacts( + [f], censoring_enabled=True, dry_run=True + ) + + # File content should be unchanged in dry run + assert f.read_text() == original + # But results should still report what would be censored + assert len(results) == 1 + assert results[0].sanitized + + def test_vault_secrets_are_censored(self, tmp_path): + f = tmp_path / "log.txt" + # Use content that won't match keyword patterns but contains a vault secret + f.write_text("Connecting to host with my-vault-secret-value as auth\n", encoding="utf-8") + + processed, results = apply_censoring_to_artifacts( + [f], + censoring_enabled=True, + vault_secrets={"my-vault-secret-value"}, + secret_mapping={"my-vault-secret-value": "test-vault/api-key"}, + ) + + content = f.read_text() + assert "my-vault-secret-value" not in content + assert "*******" in content + + def test_summary_counts_are_correct(self, tmp_path): + # Create files of each type + clean = tmp_path / "clean.txt" + clean.write_text("all good\n", encoding="utf-8") + + sensitive = tmp_path / "app.cfg" + sensitive.write_text("api_key=abc\n", encoding="utf-8") + + _, results = apply_censoring_to_artifacts( + [clean, sensitive], censoring_enabled=True + ) + + sanitized_count = len([r for r in results if r.sanitized]) + excluded_count = len([r for r in results if r.censored and not r.sanitized]) + clean_count = len([r for r in results if not r.censored]) + + assert clean_count == 1 + assert sanitized_count == 1 + assert excluded_count == 0 + + +# --------------------------------------------------------------------------- +# End-to-end directory censoring +# --------------------------------------------------------------------------- + + +class TestEndToEndDirectoryCensoring: + def test_full_directory_scan(self, tmp_path): + """Simulate the full censoring workflow on a directory tree.""" + # Create a realistic artifact tree + subdir = tmp_path / "test_run" / "artifacts" + subdir.mkdir(parents=True) + + (subdir / "pod_logs.txt").write_text("INFO: Pod started\nINFO: Pod ready\n") + (subdir / "config.yaml").write_text("password: supersecret\nhost: example.com\n") + (subdir / "results.json").write_text('{"score": 95, "status": "pass"}\n') + (subdir / "server.pem").write_text("-----BEGIN CERTIFICATE-----\nfake\n") + + # Collect all files + all_paths = list(tmp_path.rglob("*")) + file_paths = [p for p in all_paths if p.is_file()] + + processed, results = apply_censoring_to_artifacts(file_paths, censoring_enabled=True) + + # All files should be in processed (all get sanitized, none fully excluded) + assert len(processed) == 4 + + # Check specific files + config_content = (subdir / "config.yaml").read_text() + assert "supersecret" not in config_content + assert "[REDACTED]" in config_content + assert "host: example.com" in config_content + + pem_content = (subdir / "server.pem").read_text() + assert "Content censored by caliper" in pem_content + + pod_content = (subdir / "pod_logs.txt").read_text() + assert "Pod started" in pod_content # Clean content preserved + + def test_read_only_file_handling(self, tmp_path): + """Test that read-only files can be sanitized.""" + import stat + + f = tmp_path / "readonly_config.txt" + f.write_text("password=readonly_secret\n", encoding="utf-8") + f.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.sanitized + content = f.read_text() + assert "readonly_secret" not in content + assert "[REDACTED]" in content + + def test_empty_file_passes_clean(self, tmp_path): + f = tmp_path / "empty.txt" + f.write_text("", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert not result.censored + assert not result.sanitized + + def test_utf8_content_preserved(self, tmp_path): + f = tmp_path / "unicode.txt" + f.write_text("Ünîcödé content: password=test123\nMore: über cool\n", encoding="utf-8") + + censor = ArtifactCensor() + result = censor._sanitize_file_content(f) + + assert result.sanitized + content = f.read_text() + assert "test123" not in content + assert "über cool" in content From ef1dfb41b11aa602a1c74e13945e3f817ef3a399 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 28 Aug 2026 11:39:28 +0200 Subject: [PATCH 0801/1072] [caliper] tests: test_censoring: ruff --- projects/caliper/tests/test_censoring.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/projects/caliper/tests/test_censoring.py b/projects/caliper/tests/test_censoring.py index 2708df689..c12d71448 100644 --- a/projects/caliper/tests/test_censoring.py +++ b/projects/caliper/tests/test_censoring.py @@ -156,7 +156,8 @@ def test_api_secret_pattern(self): assert self._matches_any_pattern("api-secret=supersecret") def test_token_pattern(self): - assert self._matches_any_pattern("token=eyJhbGciOiJIUzI1NiJ9") + # Test specific token patterns that are enabled (general token= is disabled) + assert self._matches_any_pattern("api_token=eyJhbGciOiJIUzI1NiJ9") def test_bearer_token(self): assert self._matches_any_pattern("Bearer eyJhbGciOiJIUzI1NiJ9") @@ -390,9 +391,7 @@ def test_censoring_disabled_returns_all(self, tmp_path): f = tmp_path / "secret.pem" f.write_text("private key data\n", encoding="utf-8") - paths, results = apply_censoring_to_artifacts( - [f], censoring_enabled=False, verbose=True - ) + paths, results = apply_censoring_to_artifacts([f], censoring_enabled=False, verbose=True) assert f in paths assert len(results) == 0 @@ -434,9 +433,7 @@ def test_dry_run_preserves_all_files(self, tmp_path): original = "password=secret123\n" f.write_text(original, encoding="utf-8") - processed, results = apply_censoring_to_artifacts( - [f], censoring_enabled=True, dry_run=True - ) + processed, results = apply_censoring_to_artifacts([f], censoring_enabled=True, dry_run=True) # File content should be unchanged in dry run assert f.read_text() == original @@ -468,9 +465,7 @@ def test_summary_counts_are_correct(self, tmp_path): sensitive = tmp_path / "app.cfg" sensitive.write_text("api_key=abc\n", encoding="utf-8") - _, results = apply_censoring_to_artifacts( - [clean, sensitive], censoring_enabled=True - ) + _, results = apply_censoring_to_artifacts([clean, sensitive], censoring_enabled=True) sanitized_count = len([r for r in results if r.sanitized]) excluded_count = len([r for r in results if r.censored and not r.sanitized]) From 377364c340f3477d0dfe07f0490804488e1cf86f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 28 Aug 2026 11:53:06 +0200 Subject: [PATCH 0802/1072] address coderabbit reviews --- projects/caliper/cli/commands.py | 76 +++++++++++++++------ projects/caliper/orchestration/censoring.py | 56 +++++++-------- 2 files changed, 84 insertions(+), 48 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index cba9aedf4..76efded9e 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -1407,9 +1407,36 @@ def artifacts_censor( click.echo(f"Scanning {len(all_artifact_paths)} artifacts in {from_path}") - # Apply censoring + # Determine which files to censor based on output_path + if output_path: + # Copy source tree to output directory first, then censor the copy + if not dry_run: + try: + if output_path.exists(): + import shutil + + shutil.rmtree(output_path) + shutil.copytree(from_path, output_path) + click.echo(f"Copied source tree to {output_path}") + except Exception as e: + click.echo(f"Error copying source tree: {e}", err=True) + sys.exit(1) + else: + click.echo(f"Dry run: Would copy source tree to {output_path}") + + # Get corresponding files in the output directory for censoring + artifact_paths_to_censor = [ + output_path / p.relative_to(from_path) for p in all_artifact_paths + ] + censoring_source_path = output_path + else: + # Use original files for in-place censoring + artifact_paths_to_censor = all_artifact_paths + censoring_source_path = from_path + + # Apply censoring to the target files (copy or original) filtered_paths, censoring_results = apply_censoring_to_artifacts( - all_artifact_paths, censoring_enabled=True, verbose=verbose, dry_run=dry_run + artifact_paths_to_censor, censoring_enabled=True, verbose=verbose, dry_run=dry_run ) # Separate results: excluded (fully censored), sanitized (redacted in-place), and clean @@ -1429,13 +1456,13 @@ def artifacts_censor( click.echo("\nSanitized files (redacted in-place, kept):") for result in censoring_results: if result.sanitized: - rel_path = result.file_path.relative_to(from_path) + rel_path = result.file_path.relative_to(censoring_source_path) click.echo(f" - {rel_path}: {result.reason}") if excluded_files: click.echo("\nExcluded files (to be removed):") for result in censoring_results: if result.censored and not result.sanitized: - rel_path = result.file_path.relative_to(from_path) + rel_path = result.file_path.relative_to(censoring_source_path) click.echo(f" - {rel_path}: {result.reason}") # Generate report if requested @@ -1450,7 +1477,7 @@ def artifacts_censor( "excluded_files": len(excluded_files), "sanitized_details": [ { - "file": str(r.file_path.relative_to(from_path)), + "file": str(r.file_path.relative_to(censoring_source_path)), "reason": r.reason, } for r in censoring_results @@ -1458,7 +1485,7 @@ def artifacts_censor( ], "excluded_details": [ { - "file": str(r.file_path.relative_to(from_path)), + "file": str(r.file_path.relative_to(censoring_source_path)), "reason": r.reason, } for r in censoring_results @@ -1484,27 +1511,34 @@ def artifacts_censor( click.echo("\nDry run: No files were modified") if output_path: kept_count = len(clean_files) + len(sanitized_files) - click.echo(f"Would copy {kept_count} files to {output_path}") + click.echo( + f"Would copy source tree to {output_path} and remove {len(excluded_files)} excluded files" + ) else: click.echo(f"Would remove {len(excluded_files)} excluded files from {from_path}") elif output_path: - # Copy clean and sanitized files to output directory (exclude only truly excluded files) - kept_files = clean_files + sanitized_files - try: - output_path.mkdir(parents=True, exist_ok=True) - - for kept_file in kept_files: - rel_path = kept_file.relative_to(from_path) - dest_path = output_path / rel_path - dest_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(kept_file, dest_path) + # Files have already been copied and censored in the output directory + # Now remove excluded files from the output directory + if excluded_files: + try: + for excluded_file in excluded_files: + excluded_file.unlink() + if verbose: + rel_path = excluded_file.relative_to(output_path) + click.echo(f" Removed from output: {rel_path}") - click.echo(f"\nCopied {len(kept_files)} files to {output_path}") + kept_count = len(clean_files) + len(sanitized_files) + click.echo( + f"\nProcessed artifacts in {output_path}: {kept_count} kept, {len(excluded_files)} removed" + ) - except Exception as e: - click.echo(f"Error copying files: {e}", err=True) - sys.exit(1) + except Exception as e: + click.echo(f"Error removing excluded files from output: {e}", err=True) + sys.exit(1) + else: + kept_count = len(clean_files) + len(sanitized_files) + click.echo(f"\nProcessed artifacts in {output_path}: {kept_count} files, all clean!") else: # Remove only excluded files in-place (sanitized files are already redacted and kept) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 6e2817dc4..6d7c61ebe 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -104,12 +104,36 @@ def censor_text(text: str, verbose: bool = False) -> str: censored_text = text replacements_made = 0 + # Import keyword patterns for content censoring (always try this first) + from projects.caliper.engine.file_export.censoring_rules import ( + COMPILED_KEYWORD_PATTERNS, + KEYWORD_PATTERNS, + ) + + # Apply keyword pattern censoring first (preserve this even if vault discovery fails) + for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): + matches = list(pattern.finditer(censored_text)) + if matches: + # Replace all matched spans with redacted text, preserving other content + # Process patterns in reverse order by position to maintain string indices + for match in reversed(matches): + censored_text = ( + censored_text[: match.start()] + "[REDACTED]" + censored_text[match.end() :] + ) + replacements_made += 1 + + if verbose: + logger.info( + f"Censored {len(matches)} instances of pattern '{KEYWORD_PATTERNS[i]}' in text" + ) + + # Now try vault secrets discovery and censoring try: # Discover vault secrets for censoring vault_secrets, secret_mapping = discover_vault_secrets(verbose=verbose) # /!\ secret_mapping contains the secret values. Process with extra care. - # Replace vault secrets first (more specific) + # Replace vault secrets (more specific) for secret in vault_secrets: if secret and secret.strip() and secret.strip() in censored_text: censored_text = censored_text.replace(secret.strip(), "[REDACTED-VAULT]") @@ -118,38 +142,16 @@ def censor_text(text: str, verbose: bool = False) -> str: vault_identifier = secret_mapping.get(secret.strip(), "unknown vault") logger.info(f"Censored vault secret from {vault_identifier} in text") - # Import keyword patterns for content censoring - from projects.caliper.engine.file_export.censoring_rules import ( - COMPILED_KEYWORD_PATTERNS, - KEYWORD_PATTERNS, - ) - - # Apply keyword pattern censoring - for i, pattern in enumerate(COMPILED_KEYWORD_PATTERNS): - matches = list(pattern.finditer(censored_text)) - if matches: - # Replace all matched spans with redacted text, preserving other content - # Process patterns in reverse order by position to maintain string indices - for match in reversed(matches): - censored_text = ( - censored_text[: match.start()] + "[REDACTED]" + censored_text[match.end() :] - ) - replacements_made += 1 - - if verbose: - logger.info( - f"Censored {len(matches)} instances of pattern '{KEYWORD_PATTERNS[i]}' in text" - ) - if replacements_made > 0: logger.info(f"Censored {replacements_made} sensitive items from text") elif verbose: logger.info("No sensitive content found in text") except Exception as e: - logger.warning(f"Error during text censoring: {e}") - # On error, add a warning to the text - censored_text += "\n\n> ⚠️ **Note**: Automatic content censoring encountered an error. Please review this content manually for sensitive information." + logger.error(f"Failed to discover vault secrets during text censoring: {e}") + # Propagate the vault discovery failure after keyword censoring is complete + # This allows _censor_notification_text to handle the failure appropriately + raise return censored_text From e4ffa389e6de52280c6771c46af31537fcf671d9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 28 Aug 2026 11:53:13 +0200 Subject: [PATCH 0803/1072] address coderabbit reviews --- projects/core/library/export_notifications.py | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 0be57b464..ed6c16dbc 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -32,8 +32,17 @@ def _censor_notification_text(text: str, verbose: bool = False) -> str: Returns: Censored notification text with sensitive content replaced + + Raises: + Exception: If vault discovery fails, preventing notification delivery """ - return censor_text(text, verbose=verbose) + try: + return censor_text(text, verbose=verbose) + except Exception as e: + logger.error(f"Censoring failed during notification preparation: {e}") + # Re-raise to abort GitHub and Slack notification delivery + # rather than sending potentially uncensored text + raise def send_notification( @@ -63,7 +72,11 @@ def send_notification( ) # Apply censoring to notification content before sending - notification_status = _censor_notification_text(notification_status, verbose=dry_run) + try: + notification_status = _censor_notification_text(notification_status, verbose=dry_run) + except Exception as e: + logger.error(f"Notification censoring failed, aborting notification delivery: {e}") + return False # Send actual notifications if dry_run: @@ -125,23 +138,31 @@ def send_notification( artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None # Censor status fields before creating NotificationContext - censored_status = yaml.safe_load( - _censor_notification_text(yaml.dump(status), verbose=dry_run) - ) - context = NotificationContext( - status=censored_status, - finish_reason=str(finish_reason), - project_name=project or "unknown", - pr_number=os.environ.get("PULL_NUMBER"), - job_type=os.environ.get("JOB_TYPE"), - artifact_dir=artifact_dir, - ) - ok = notification_provider.notify(context) - if ok: - logger.info("Successfully sent per-project Slack notification") - else: - logger.warning("Per-project Slack notification failed") + try: + censored_status = yaml.safe_load( + _censor_notification_text(yaml.dump(status), verbose=dry_run) + ) + except Exception as e: + logger.error( + f"Slack notification censoring failed, aborting Slack notification: {e}" + ) notification_success = False + else: + # Only proceed with Slack notification if censoring succeeded + context = NotificationContext( + status=censored_status, + finish_reason=str(finish_reason), + project_name=project or "unknown", + pr_number=os.environ.get("PULL_NUMBER"), + job_type=os.environ.get("JOB_TYPE"), + artifact_dir=artifact_dir, + ) + ok = notification_provider.notify(context) + if ok: + logger.info("Successfully sent per-project Slack notification") + else: + logger.warning("Per-project Slack notification failed") + notification_success = False except Exception as e: logger.warning(f"Failed to send per-project Slack notification: {e}") notification_success = False From b8916c0a3c0e7e4d28e472f6843d233203dfd826 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 7 Sep 2026 21:07:42 +0200 Subject: [PATCH 0804/1072] feat: Add censor_text support for vault secrets - Add censor_text field to VaultContent dataclass - Update vault loading to parse censor_text from YAML - Modify censoring logic to use censor_text when available instead of [REDACTED-VAULT] - Skip reporting when using censor_text (expected behavior) - Remove legacy vault content format support - Update all censoring functions to handle censor_text_mapping This allows vault definitions to specify custom replacement text for secrets, making censored logs more meaningful while still protecting sensitive data. Co-Authored-By: Claude Sonnet 4 --- .../caliper/engine/file_export/censoring.py | 19 ++- projects/caliper/orchestration/censoring.py | 112 +++++++++++++----- projects/core/library/vault.py | 24 ++-- vaults/psap-forge-mlflow-export.yaml | 1 + 4 files changed, 114 insertions(+), 42 deletions(-) diff --git a/projects/caliper/engine/file_export/censoring.py b/projects/caliper/engine/file_export/censoring.py index 6aead82db..1f5381c92 100644 --- a/projects/caliper/engine/file_export/censoring.py +++ b/projects/caliper/engine/file_export/censoring.py @@ -72,6 +72,7 @@ def __init__( self, vault_secrets: set[str] | None = None, secret_mapping: dict[str, str] | None = None, + censor_text_mapping: dict[str, str] | None = None, verbose: bool = False, dry_run: bool = False, ): @@ -81,11 +82,13 @@ def __init__( Args: vault_secrets: Set of secret strings loaded from vaults secret_mapping: Dict mapping secret strings to vault/content identifiers + censor_text_mapping: Dict mapping secret strings to censor_text replacements verbose: Enable verbose logging dry_run: Skip file modifications while preserving analysis """ self.vault_secrets = vault_secrets or set() self.secret_mapping = secret_mapping or {} + self.censor_text_mapping = censor_text_mapping or {} self.verbose = verbose self.dry_run = dry_run @@ -201,10 +204,15 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: # Always check for vault secrets, independent of keyword pattern detection for secret in self.vault_secrets: if secret and secret.strip() and secret.strip() in content: - content = content.replace(secret.strip(), "*******") + # Use censor_text if available, otherwise use [REDACTED-VAULT] + replacement = self.censor_text_mapping.get(secret.strip(), "[REDACTED-VAULT]") + content = content.replace(secret.strip(), replacement) sanitized = True - vault_identifier = self.secret_mapping.get(secret.strip(), "unknown vault") - reasons.append(f"contains vault secret: {vault_identifier}") + + # Only add to reasons if not using censor_text (unexpected vault content) + if secret.strip() not in self.censor_text_mapping: + vault_identifier = self.secret_mapping.get(secret.strip(), "unknown vault") + reasons.append(f"contains vault secret: {vault_identifier}") if sanitized: # Write sanitized content back to original file (skip if dry run) @@ -277,6 +285,7 @@ def apply_censoring_to_artifacts( verbose: bool = False, vault_secrets: set[str] | None = None, secret_mapping: dict[str, str] | None = None, + censor_text_mapping: dict[str, str] | None = None, dry_run: bool = False, ) -> tuple[list[Path], list[CensoringResult]]: """ @@ -284,7 +293,7 @@ def apply_censoring_to_artifacts( This function processes artifacts by: - Replacing sensitive content patterns with "Content censored by caliper" in-place - - Replacing vault secrets with "*******" in-place + - Replacing vault secrets with "*******" or censor_text if available in-place - Excluding files with sensitive filename patterns (.pem, .key, files with "secret" in name, etc.) Args: @@ -293,6 +302,7 @@ def apply_censoring_to_artifacts( verbose: Enable verbose logging vault_secrets: Set of vault secret strings to censor secret_mapping: Dict mapping secret strings to vault/content identifiers + censor_text_mapping: Dict mapping secret strings to censor_text replacements dry_run: Skip file modifications while preserving analysis Returns: @@ -309,6 +319,7 @@ def apply_censoring_to_artifacts( censor = ArtifactCensor( vault_secrets=vault_secrets or set(), secret_mapping=secret_mapping or {}, + censor_text_mapping=censor_text_mapping or {}, verbose=verbose, dry_run=dry_run, ) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 6d7c61ebe..74d5e0999 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -21,7 +21,9 @@ logger = logging.getLogger(__name__) -def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, str]]: +def discover_vault_secrets( + verbose: bool = False, +) -> tuple[set[str], dict[str, str], dict[str, str]]: """ Discover all vault secrets for censoring (all vault content is treated as sensitive). @@ -29,12 +31,14 @@ def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, s verbose: Enable verbose logging Returns: - Tuple of (secret_strings, secret_mapping) where: + Tuple of (secret_strings, secret_mapping, censor_text_mapping) where: - secret_strings: Set of secret strings loaded from vaults - secret_mapping: Dict mapping secret string -> vault/content identifier + - censor_text_mapping: Dict mapping secret string -> censor_text replacement (if available) """ vault_secrets = set() secret_mapping = {} + censor_text_mapping = {} try: vault_manager = vault_lib.get_vault_manager() @@ -58,24 +62,67 @@ def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, s ) continue - content_path = vault_manager.get_vault_content_path(vault_name, content_name) - if not (content_path and content_path.exists()): - logger.warning(f"Invalid vault found: {vault_name} {content_name} (missing)") - continue - - try: - # Read vault content (assume it's text) - content_text = content_path.read_text(encoding="utf-8", errors="ignore").strip() - if not content_text: - logger.warning(f"Invalid vault found: {vault_name} {content_name} (empty)") - else: - vault_secrets.add(content_text) - secret_mapping[content_text] = f"{vault_name}/{content_name}" - secrets_discovered += 1 - if verbose: - logger.info(f"Discovered secret from vault {vault_name}/{content_name}") - except Exception as e: - logger.warning(f"Failed to read vault content {vault_name}/{content_name}: {e}") + # If censor_text is available, use it directly instead of reading file + if content_def.censor_text: + content_path = vault_manager.get_vault_content_path(vault_name, content_name) + if not (content_path and content_path.exists()): + logger.warning( + f"Invalid vault found: {vault_name} {content_name} (missing)" + ) + continue + + try: + # Read actual vault content + content_text = content_path.read_text( + encoding="utf-8", errors="ignore" + ).strip() + if not content_text: + logger.warning( + f"Invalid vault found: {vault_name} {content_name} (empty)" + ) + else: + vault_secrets.add(content_text) + secret_mapping[content_text] = f"{vault_name}/{content_name}" + censor_text_mapping[content_text] = content_def.censor_text + secrets_discovered += 1 + if verbose: + logger.info( + f"Discovered secret from vault {vault_name}/{content_name} with censor_text" + ) + except Exception as e: + logger.warning( + f"Failed to read vault content {vault_name}/{content_name}: {e}" + ) + else: + # No censor_text, read file content for traditional censoring + content_path = vault_manager.get_vault_content_path(vault_name, content_name) + if not (content_path and content_path.exists()): + logger.warning( + f"Invalid vault found: {vault_name} {content_name} (missing)" + ) + continue + + try: + # Read vault content (assume it's text) + content_text = content_path.read_text( + encoding="utf-8", errors="ignore" + ).strip() + if not content_text: + logger.warning( + f"Invalid vault found: {vault_name} {content_name} (empty)" + ) + else: + vault_secrets.add(content_text) + secret_mapping[content_text] = f"{vault_name}/{content_name}" + secrets_discovered += 1 + if verbose: + logger.info( + f"Discovered secret from vault {vault_name}/{content_name}" + ) + except Exception as e: + logger.warning( + f"Failed to read vault content {vault_name}/{content_name}: {e}" + ) if verbose: logger.info(f"Discovered {secrets_discovered} vault secrets for censoring") @@ -84,7 +131,7 @@ def discover_vault_secrets(verbose: bool = False) -> tuple[set[str], dict[str, s logger.exception(f"Failed to discover vault secrets: {e}") raise - return vault_secrets, secret_mapping + return vault_secrets, secret_mapping, censor_text_mapping def censor_text(text: str, verbose: bool = False) -> str: @@ -130,17 +177,25 @@ def censor_text(text: str, verbose: bool = False) -> str: # Now try vault secrets discovery and censoring try: # Discover vault secrets for censoring - vault_secrets, secret_mapping = discover_vault_secrets(verbose=verbose) + vault_secrets, secret_mapping, censor_text_mapping = discover_vault_secrets(verbose=verbose) # /!\ secret_mapping contains the secret values. Process with extra care. # Replace vault secrets (more specific) for secret in vault_secrets: if secret and secret.strip() and secret.strip() in censored_text: - censored_text = censored_text.replace(secret.strip(), "[REDACTED-VAULT]") - replacements_made += 1 - if verbose: + # Use censor_text if available, otherwise use [REDACTED-VAULT] + replacement = censor_text_mapping.get(secret.strip(), "[REDACTED-VAULT]") + censored_text = censored_text.replace(secret.strip(), replacement) + + # Only count as replacement and report if not using censor_text (unexpected vault content) + if secret.strip() not in censor_text_mapping: + replacements_made += 1 + if verbose: + vault_identifier = secret_mapping.get(secret.strip(), "unknown vault") + logger.info(f"Censored vault secret from {vault_identifier} in text") + elif verbose: vault_identifier = secret_mapping.get(secret.strip(), "unknown vault") - logger.info(f"Censored vault secret from {vault_identifier} in text") + logger.info(f"Applied censor_text replacement for {vault_identifier} in text") if replacements_made > 0: logger.info(f"Censored {replacements_made} sensitive items from text") @@ -178,7 +233,9 @@ def orchestration_apply_censoring( return False # Discover vault secrets - vault_secrets, secret_mapping = discover_vault_secrets(verbose=export_cfg.verbose) + vault_secrets, secret_mapping, censor_text_mapping = discover_vault_secrets( + verbose=export_cfg.verbose + ) # /!\ secret_mapping contains the secret values. Process with extra care. # Collect all artifact files @@ -195,6 +252,7 @@ def orchestration_apply_censoring( verbose=export_cfg.verbose, vault_secrets=vault_secrets, secret_mapping=secret_mapping, + censor_text_mapping=censor_text_mapping, ) # Separate results by type diff --git a/projects/core/library/vault.py b/projects/core/library/vault.py index bf270501e..262da77f5 100644 --- a/projects/core/library/vault.py +++ b/projects/core/library/vault.py @@ -32,6 +32,7 @@ class VaultContent: description: str filename: str | None = None sensible: bool = True + censor_text: str | None = None _vault: Optional["VaultDefinition"] = None def __post_init__(self): @@ -109,19 +110,20 @@ def _load_vault_definition(self, vault_file: Path) -> VaultDefinition: # Parse content definitions content = {} for content_name, content_def in data.get("content", {}).items(): - if isinstance(content_def, dict): - # New format with file mapping and description - filename = content_def.get("file", content_name) - description = content_def.get("description", "") # Don't provide default - sensible = content_def.get("sensible", True) # Default to True - else: - # Legacy format - content_def is the description - filename = content_name - description = content_def if content_def else "" - sensible = True # Sensible by default + if not isinstance(content_def, dict): + raise ValueError(f"Vault content '{content_name}' must be a dictionary") + + filename = content_def.get("file", content_name) + description = content_def.get("description", "") + sensible = content_def.get("sensible", True) + censor_text = content_def.get("censor_text") content[content_name] = VaultContent( - name=content_name, description=description, filename=filename, sensible=sensible + name=content_name, + description=description, + filename=filename, + sensible=sensible, + censor_text=censor_text, ) vault_def = VaultDefinition( diff --git a/vaults/psap-forge-mlflow-export.yaml b/vaults/psap-forge-mlflow-export.yaml index c4c2425e1..6fb3e2208 100644 --- a/vaults/psap-forge-mlflow-export.yaml +++ b/vaults/psap-forge-mlflow-export.yaml @@ -13,6 +13,7 @@ content: insecure_tls: true tracking_uri: + censor_text: "https://mlflow_tracking_uri_hidden" description: | the URI of the MLFlow endpoint, so that the CI engine can censor it in the logs From a57f8ba3da1597a3aa7cc9cd229e98e4a7b4bf15 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 7 Sep 2026 22:28:24 +0200 Subject: [PATCH 0805/1072] [caliper] engine: file_export: censoring_rules: exlucde version specifiers --- projects/caliper/engine/file_export/censoring_rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index 1c56b0bf7..8de74e4cc 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -19,10 +19,11 @@ r"api[_-]?key\s*[:=]\s*\S+", r"apikey\s*[:=]\s*\S+", r"api[_-]?secret\s*[:=]\s*\S+", - # Token patterns - r"token\s*[:=]\s*\S+", + # Token patterns (exclude version specifiers like ==0.14.0) + r"(?:^|[^a-zA-Z])token\s*[:=]\s*(?![=~<>])\S+", r"secret[_-]?token\s*[:=]\s*\S+", r"access[_-]?token\s*[:=]\s*\S+", + r"api[_-]?token\s*[:=]\s*\S+", r"refresh[_-]?token\s*[:=]\s*\S+", # Bearer tokens (compiled with IGNORECASE, so one pattern covers both cases) r"Bearer\s+[A-Za-z0-9+/=]+", From 4b5cdc57ad377ff913ae1498ed51e1d282da8743 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 10:20:11 +0200 Subject: [PATCH 0806/1072] [caliper] orchestration: censoring: improve the notification message --- projects/caliper/orchestration/censoring.py | 28 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 74d5e0999..65ddb0e36 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -393,15 +393,37 @@ def format_file_list(files, limit=10): ] if sanitized_files: + # Analyze what types of censoring occurred by examining reasons + sanitized_results = [r for r in censoring_results if r.sanitized] + reasons = [r.reason for r in sanitized_results] + + # Categorize types of censoring that occurred + has_sanitized_replacements = any( + "sensitive content detected" == reason for reason in reasons + ) + has_placeholder_replacements = any( + "sensitive content detected" != reason for reason in reasons + ) + message_parts.append( f""" {len(sanitized_files)} file(s) had sensitive content sanitized and included in export: -{format_file_list(sanitized_files)} - -Sensitive content (passwords, API keys, tokens) was replaced with placeholder text.""" +{format_file_list(sanitized_files)}""" ) + # Generate appropriate description based on censoring types + if has_sanitized_replacements and has_placeholder_replacements: + message_parts.append( + "Some sensitive content was replaced with sanitized values, and other sensitive content (passwords, API keys, tokens) was replaced with placeholder text." + ) + elif has_sanitized_replacements and not has_placeholder_replacements: + message_parts.append("Sensitive content was replaced with sanitized values.") + else: + message_parts.append( + "Sensitive content (passwords, API keys, tokens) was replaced with placeholder text." + ) + if excluded_files: message_parts.append( f""" From dcca2ec8d1763f1b4821cb142e904ff2d814e87c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 10:23:33 +0200 Subject: [PATCH 0807/1072] [core] library: export_notifications: ignore the lost+found dir in the steps --- projects/core/library/export_notifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index ed6c16dbc..95c940676 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -335,7 +335,7 @@ def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | No step_status = [] for step_dir in sorted(artifact_dir.glob("*")): - if not step_dir.is_dir() or step_dir.name.startswith("."): + if not step_dir.is_dir() or step_dir.name.startswith(".") or step_dir.name == "lost+found": continue step_name = step_dir.name From 19e97d0f474592bddad8f374204da81efb736ed2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 10:25:00 +0200 Subject: [PATCH 0808/1072] [core] library: export_notifications: add custom emoji for the export-artifacts step --- projects/core/library/export_notifications.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 95c940676..72be9d821 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -359,6 +359,9 @@ def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | No except Exception: exit_status_emoji = "❓" + if step_dir.name.endswith("__export-artifacts"): + exit_status_emoji = "📤" + # Count ERROR and WARNING messages in run.log log_counts = _count_log_messages(step_dir) log_summary = _format_log_summary(log_counts) @@ -701,18 +704,18 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: if isinstance(step_data, dict): status = step_data.get("status", "unknown") status_emoji = ( - "✅" if status == "success" else "❌" if status == "failed" else "⚪" + "✔️" if status == "success" else "❌" if status == "failed" else "⚪" ) step_statuses.append(f"{status_emoji} {step_name}") # Format overall line overall_emoji = "✅" if overall_success else "❌" if step_statuses: - steps_str = " " + " • ".join(step_statuses) + steps_str = " " + " | ".join(step_statuses) else: steps_str = f" {final_status}" - postprocess_info_lines.append(f"**{dir_name}**: {overall_emoji}{steps_str}") + postprocess_info_lines.append(f"{overall_emoji} **{dir_name}**: {overall_emoji}{steps_str}") except Exception as e: postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") From 767e0df7be129fe4f03381c425755561d209fa98 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 11:20:40 +0200 Subject: [PATCH 0809/1072] [core] library: export_notifications: improve the title --- projects/core/library/export_notifications.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 72be9d821..6772283a0 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -231,17 +231,16 @@ def _build_enhanced_notification( if finish_reason == "failed": status_emoji = "❌" - base_status = f"**{status_emoji} Execution of `{fjob_project}` {fjob_args_str} {status_emoji}**" - notification_parts = [base_status] + base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {status_emoji}" + notification_parts = [base_status, ""] # Add job abort message right below overall status if applicable shutdown_status = status.get("job_shutdown") if shutdown_status and shutdown_status.get("is_aborted"): + notification_parts.append("---") shutdown_value = shutdown_status.get("shutdown_value", "Stop") notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") - - notification_parts.append("---") - notification_parts.append("") + notification_parts += ["", "---", ""] execution_engine_config = _get_execution_engine_config() if execution_engine_config: @@ -715,7 +714,7 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: else: steps_str = f" {final_status}" - postprocess_info_lines.append(f"{overall_emoji} **{dir_name}**: {overall_emoji}{steps_str}") + postprocess_info_lines.append(f"{overall_emoji} **{dir_name}**: {steps_str}") except Exception as e: postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") From e8aac2f9071f79ee07469ae86e12800e4acbffaf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 11:31:19 +0200 Subject: [PATCH 0810/1072] [caliper] properly track the censor replacement --- .../caliper/engine/file_export/censoring.py | 37 +++++- projects/caliper/orchestration/censoring.py | 121 ++++++++++++------ projects/caliper/tests/test_censoring.py | 2 +- 3 files changed, 117 insertions(+), 43 deletions(-) diff --git a/projects/caliper/engine/file_export/censoring.py b/projects/caliper/engine/file_export/censoring.py index 1f5381c92..9ff6b4841 100644 --- a/projects/caliper/engine/file_export/censoring.py +++ b/projects/caliper/engine/file_export/censoring.py @@ -55,6 +55,7 @@ class CensoringResult: censored: bool reason: str sanitized: bool = False # True if content was sanitized in-place + safely_redacted: bool = False # True if redacted using censor_text (expected) def __str__(self): if self.sanitized: @@ -158,12 +159,16 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: file_path.write_text(censored_content, encoding="utf-8") return CensoringResult( - file_path, True, f"sensitive filename pattern: {file_path.name}", sanitized=True + file_path, + True, + f"sensitive filename pattern: {file_path.name}", + sanitized=True, + safely_redacted=False, ) # Skip non-text files to avoid reading binary content if not self._is_text_file(file_path): - return CensoringResult(file_path, False, "non-text file") + return CensoringResult(file_path, False, "non-text file", safely_redacted=False) content = file_path.read_text(encoding="utf-8", errors="ignore") sanitized = False @@ -201,6 +206,10 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: sanitized = True + # Track whether only safe censor_text replacements were made + has_safe_replacements = False + has_unsafe_replacements = False + # Always check for vault secrets, independent of keyword pattern detection for secret in self.vault_secrets: if secret and secret.strip() and secret.strip() in content: @@ -209,12 +218,24 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: content = content.replace(secret.strip(), replacement) sanitized = True - # Only add to reasons if not using censor_text (unexpected vault content) + # Track type of replacement and add appropriate reason if secret.strip() not in self.censor_text_mapping: + # Unexpected vault content - add to reasons for reporting + has_unsafe_replacements = True vault_identifier = self.secret_mapping.get(secret.strip(), "unknown vault") reasons.append(f"contains vault secret: {vault_identifier}") + else: + # Expected vault content with censor_text - add specific reason + has_safe_replacements = True + vault_identifier = self.secret_mapping.get(secret.strip(), "unknown vault") + reasons.append(f"replaced with censor_text from {vault_identifier}") if sanitized: + # Determine if this was safely redacted (only censor_text, no keywords/unexpected vault) + safely_redacted = ( + has_safe_replacements and not has_unsafe_replacements and not keyword_detected + ) + # Write sanitized content back to original file (skip if dry run) if not self.dry_run: try: @@ -230,13 +251,17 @@ def _sanitize_file_content(self, file_path: Path) -> CensoringResult: raise reason = reasons[0] if reasons else "sensitive content detected" - return CensoringResult(file_path, True, reason, sanitized=True) + return CensoringResult( + file_path, True, reason, sanitized=True, safely_redacted=safely_redacted + ) - return CensoringResult(file_path, False, "content check passed") + return CensoringResult(file_path, False, "content check passed", safely_redacted=False) except Exception as e: logger.warning(f"Error sanitizing file {file_path}: {e}") - return CensoringResult(file_path, True, f"sanitization failed: {e}") + return CensoringResult( + file_path, True, f"sanitization failed: {e}", safely_redacted=False + ) def censor_files(self, file_paths: list[Path]) -> tuple[list[Path], list[CensoringResult]]: """ diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 65ddb0e36..bf8cc6023 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -225,7 +225,7 @@ def orchestration_apply_censoring( disable_censoring: Whether to skip censoring Returns: - True if any files were censored (sanitized or excluded), False otherwise + True if unexpected censoring issues occurred (excluding safe censor_text), False otherwise """ if disable_censoring: if export_cfg.verbose: @@ -266,34 +266,65 @@ def orchestration_apply_censoring( _generate_censoring_report(censoring_results, from_path) - # Abort export if any files were excluded (fail closed) + # Generate notification for excluded files but allow export to proceed if excluded_files: - # Create notification about censoring activity blocking export - _create_censoring_notification(censoring_results, from_path, export_blocked=True) + # Create notification about censoring activity + _create_censoring_notification(censoring_results, from_path, export_blocked=False) excluded_file_names = [str(r.file_path.relative_to(from_path)) for r in excluded_files] - logger.error( - f"Export aborted: {len(excluded_files)} files contain sensitive content that could not be sanitized: {excluded_file_names[:5]}{'...' if len(excluded_file_names) > 5 else ''}" - ) - raise RuntimeError( - f"Export blocked due to {len(excluded_files)} unsanitizable sensitive files" + logger.warning( + f"Export proceeding despite {len(excluded_files)} files with sensitive content that could not be sanitized: {excluded_file_names[:5]}{'...' if len(excluded_file_names) > 5 else ''}" ) + # Note: Export continues despite excluded files - notifications will alert about issues # Generate censoring report and notification if any files were processed - if sanitized_files: - # Create notification about censoring activity (export proceeds with sanitized files) - _create_censoring_notification(censoring_results, from_path, export_blocked=False) + if sanitized_files or excluded_files: + # Only create notification for unexpected censoring, not safe censor_text replacements + # (excluded files already got their notification above) + if sanitized_files: + unexpected_sanitized_files = [ + r for r in censoring_results if r.sanitized and not r.safely_redacted + ] + if unexpected_sanitized_files: + # Create notification about unexpected censoring activity + _create_censoring_notification( + unexpected_sanitized_files, from_path, export_blocked=False + ) + else: if export_cfg.verbose: - logger.info( - f"Export proceeding with {len(sanitized_files)} files sanitized for sensitive content" + logger.info("No files required censoring") + return False # No censoring occurred + + if export_cfg.verbose and (sanitized_files or excluded_files): + # Calculate counts for verbose logging + unexpected_sanitized_count = len( + [r for r in censoring_results if r.sanitized and not r.safely_redacted] + ) + safe_files = len(sanitized_files) - unexpected_sanitized_count + + log_parts = [] + if unexpected_sanitized_count > 0: + log_parts.append( + f"{unexpected_sanitized_count} files with unexpected sensitive content sanitized" ) - return True # Censoring occurred + if safe_files > 0: + log_parts.append(f"{safe_files} files with expected vault content replaced") + if excluded_files: + log_parts.append(f"{len(excluded_files)} files excluded due to sensitive filenames") - elif export_cfg.verbose: - logger.info("No files required censoring") + if log_parts: + logger.info(f"Export proceeding: {', '.join(log_parts)}") + else: + logger.info("Export proceeding") - return False # No censoring occurred + # Only return True (censoring occurred) if there were unexpected issues + # Safe censor_text replacements should not trigger exit code 1 + has_unexpected_issues = ( + len([r for r in censoring_results if r.sanitized and not r.safely_redacted]) > 0 + or len(excluded_files) > 0 + ) + return has_unexpected_issues def _generate_censoring_report(censoring_results, from_path: Path) -> None: @@ -301,29 +332,51 @@ def _generate_censoring_report(censoring_results, from_path: Path) -> None: try: from datetime import datetime - # Group censored files by reason - censored_by_reason = {} + # Separate safe vs unexpected censoring + safe_censoring_by_reason = {} + unexpected_censoring_by_reason = {} + for r in censoring_results: if r.censored: reason = r.reason - if reason not in censored_by_reason: - censored_by_reason[reason] = [] - file_path = ( str(r.file_path.relative_to(from_path)) if r.file_path.is_relative_to(from_path) else str(r.file_path) ) - censored_by_reason[reason].append(file_path) + + # Use the safely_redacted flag to categorize + if r.safely_redacted: + # Safe censoring - expected vault content with censor_text + if reason not in safe_censoring_by_reason: + safe_censoring_by_reason[reason] = [] + safe_censoring_by_reason[reason].append(file_path) + else: + # Unexpected censoring - unexpected sensitive content + if reason not in unexpected_censoring_by_reason: + unexpected_censoring_by_reason[reason] = [] + unexpected_censoring_by_reason[reason].append(file_path) + + # Count different types of censoring + safe_censored_files = sum(len(files) for files in safe_censoring_by_reason.values()) + unexpected_censored_files = sum( + len(files) for files in unexpected_censoring_by_reason.values() + ) + total_censored_files = safe_censored_files + unexpected_censored_files # Create report data report_data = { "timestamp": datetime.now().isoformat(), "source_directory": str(from_path), "total_files": len(censoring_results), - "censored_files": len([r for r in censoring_results if r.censored]), "clean_files": len([r for r in censoring_results if not r.censored]), - "censored_by_reason": censored_by_reason, + "censored_files": total_censored_files, + "safe_censored_files": safe_censored_files, + "unexpected_censored_files": unexpected_censored_files, + "safe_censoring_by_reason": safe_censoring_by_reason, + "unexpected_censoring_by_reason": unexpected_censoring_by_reason, + # Keep old format for compatibility + "censored_by_reason": {**safe_censoring_by_reason, **unexpected_censoring_by_reason}, } # Write report to ARTIFACT_DIR @@ -347,12 +400,13 @@ def _create_censoring_notification( ) -> None: """Create a notification file about censoring activity.""" try: - # Separate results + # censoring_results now contains only unexpected/problematic censoring results + # (safe censor_text replacements are excluded) sanitized_files = [r.file_path for r in censoring_results if r.sanitized] excluded_files = [r.file_path for r in censoring_results if r.censored and not r.sanitized] if not sanitized_files and not excluded_files: - return # No censoring occurred + return # No unexpected censoring occurred # Prepare file lists def format_file_list(files, limit=10): @@ -393,17 +447,12 @@ def format_file_list(files, limit=10): ] if sanitized_files: - # Analyze what types of censoring occurred by examining reasons + # Analyze what types of censoring occurred using the safely_redacted flag sanitized_results = [r for r in censoring_results if r.sanitized] - reasons = [r.reason for r in sanitized_results] # Categorize types of censoring that occurred - has_sanitized_replacements = any( - "sensitive content detected" == reason for reason in reasons - ) - has_placeholder_replacements = any( - "sensitive content detected" != reason for reason in reasons - ) + has_sanitized_replacements = any(r.safely_redacted for r in sanitized_results) + has_placeholder_replacements = any(not r.safely_redacted for r in sanitized_results) message_parts.append( f""" diff --git a/projects/caliper/tests/test_censoring.py b/projects/caliper/tests/test_censoring.py index c12d71448..6fd9d7d60 100644 --- a/projects/caliper/tests/test_censoring.py +++ b/projects/caliper/tests/test_censoring.py @@ -455,7 +455,7 @@ def test_vault_secrets_are_censored(self, tmp_path): content = f.read_text() assert "my-vault-secret-value" not in content - assert "*******" in content + assert "[REDACTED-VAULT]" in content def test_summary_counts_are_correct(self, tmp_path): # Create files of each type From 641a250f8ee23da8bfa9c9ecac10f68f9f81f769 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 11:53:21 +0200 Subject: [PATCH 0811/1072] [caliper] engine: file_export: censoring_rules disable the token= rule, too sensitive --- projects/caliper/engine/file_export/censoring_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index 8de74e4cc..1c3d695d9 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -20,7 +20,7 @@ r"apikey\s*[:=]\s*\S+", r"api[_-]?secret\s*[:=]\s*\S+", # Token patterns (exclude version specifiers like ==0.14.0) - r"(?:^|[^a-zA-Z])token\s*[:=]\s*(?![=~<>])\S+", + # r"(?:^|[^a-zA-Z])token\s*[:=]\s*(?![=~<>])\S+", # DISABLED, too sensitive for inference work ... r"secret[_-]?token\s*[:=]\s*\S+", r"access[_-]?token\s*[:=]\s*\S+", r"api[_-]?token\s*[:=]\s*\S+", From fa7e6b69c7da0ab7d213f143c39780916fc0925b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 13:04:27 +0200 Subject: [PATCH 0812/1072] [caliper] improve the censoring notification --- projects/caliper/orchestration/censoring.py | 13 ++- projects/core/library/export_notifications.py | 81 ++++++++++++++++++- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index bf8cc6023..e05eab431 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -362,21 +362,18 @@ def _generate_censoring_report(censoring_results, from_path: Path) -> None: unexpected_censored_files = sum( len(files) for files in unexpected_censoring_by_reason.values() ) - total_censored_files = safe_censored_files + unexpected_censored_files + clean_files_count = len([r for r in censoring_results if not r.censored]) # Create report data report_data = { "timestamp": datetime.now().isoformat(), "source_directory": str(from_path), - "total_files": len(censoring_results), - "clean_files": len([r for r in censoring_results if not r.censored]), - "censored_files": total_censored_files, + "total_files": clean_files_count + safe_censored_files + unexpected_censored_files, + "clean_files": clean_files_count, + "censored_files": unexpected_censored_files, # Only unexpected censoring "safe_censored_files": safe_censored_files, - "unexpected_censored_files": unexpected_censored_files, "safe_censoring_by_reason": safe_censoring_by_reason, - "unexpected_censoring_by_reason": unexpected_censoring_by_reason, - # Keep old format for compatibility - "censored_by_reason": {**safe_censoring_by_reason, **unexpected_censoring_by_reason}, + "censored_by_reason": unexpected_censoring_by_reason, # Only unexpected censoring } # Write report to ARTIFACT_DIR diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 6772283a0..705d88dde 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -281,6 +281,14 @@ def _build_enhanced_notification( notification_parts.append("**Post-processing Status** ✅") notification_parts.extend(postprocess_status_links) + # Add censoring report section if available + censoring_report_section = _get_censoring_report_section(artifact_dir) + if censoring_report_section: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Censoring Report**") + notification_parts.extend(censoring_report_section) + except Exception as e: logger.exception(f"Failed to build the extended notifications: {e}") notification_parts.append("**Artifact Links:** Error extracting links") @@ -292,6 +300,77 @@ def _build_enhanced_notification( return "\n".join(notification_parts), notification_success +def _get_censoring_report_section(artifact_dir: Path) -> list[str] | None: + """ + Parse censoring_report.yaml if it exists and return notification section. + + Returns: + List of notification lines or None if no report exists + """ + if not artifact_dir: + return None + + # Look for censoring_report.yaml at the top level of the step files + censoring_report_path = artifact_dir / "censoring_report.yaml" + + if not censoring_report_path.exists(): + return None + + try: + with open(censoring_report_path) as f: + report_data = yaml.safe_load(f) + + if not report_data: + return None + + notification_lines = [] + + # Show number of files scanned + total_files = report_data.get("total_files", 0) + clean_files = report_data.get("clean_files", 0) + safe_censored_files = report_data.get("safe_censored_files", 0) + censored_files = report_data.get("censored_files", 0) # Only unexpected now + + notification_lines.append(f"📊 **Files scanned:** {total_files}") + notification_lines.append( + f"✅ Clean: {clean_files}, 🔒 Safe replacements: {safe_censored_files}" + ) + + # Show unexpected censoring issues if any + censored_by_reason = report_data.get("censored_by_reason", {}) + if censored_files > 0 and censored_by_reason: + notification_lines.append( + f"⚠️ **Unexpected sensitive content:** {censored_files} file(s)" + ) + notification_lines.append("") + + for reason, files in censored_by_reason.items(): + # Shorten long reasons for notification + short_reason = reason + if len(reason) > 60: + short_reason = reason[:57] + "..." + + file_count = len(files) + if file_count <= 3: + # Show individual files for small counts + file_list = ", ".join([f"`{f}`" for f in files]) + notification_lines.append(f"* {short_reason}: {file_list}") + else: + # Show count for large lists + first_files = ", ".join([f"`{f}`" for f in files[:2]]) + notification_lines.append( + f"* {short_reason}: {first_files} and {file_count - 2} more" + ) + elif censored_files == 0: + notification_lines.append("✅ **No unexpected sensitive content found**") + + return notification_lines + + except Exception as e: + logger.warning(f"Failed to parse censoring report: {e}") + return [f"⚠️ **Censoring report parsing failed:** {e}"] + + def _get_execution_engine_config() -> str | None: """Get execution engine configuration for notification.""" try: @@ -714,7 +793,7 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: else: steps_str = f" {final_status}" - postprocess_info_lines.append(f"{overall_emoji} **{dir_name}**: {steps_str}") + postprocess_info_lines.append(f"* {overall_emoji} **{dir_name}**: {steps_str}") except Exception as e: postprocess_info_lines.append(f"**{postprocess_file.name}**: Error reading file - {e}") From 1d5f141db4de9445dd5f87b29e3596f11b538f7a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 13:39:38 +0200 Subject: [PATCH 0813/1072] [core] library: export_notifications: move the imports to the top --- projects/core/library/export_notifications.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 705d88dde..203a55747 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -18,6 +18,8 @@ from projects.core.library import ci as ci_lib from projects.core.library import config, env from projects.core.library.step_status import StepStatus +from projects.core.notifications.provider import NotificationContext +from projects.core.notifications.send import send_notification as send_github_notification logger = logging.getLogger(__name__) @@ -102,8 +104,6 @@ def send_notification( # Actually send notification through GitHub API try: - from projects.core.notifications.send import send_notification as send_github_notification - # Get notification vault from configuration notification_vault = None try: @@ -134,8 +134,6 @@ def send_notification( if notification_provider: if not dry_run: try: - from projects.core.notifications.provider import NotificationContext - artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None # Censor status fields before creating NotificationContext try: @@ -551,10 +549,10 @@ def _get_postprocess_status_links( shutdown_status = status_data["job_shutdown"] status_data["job_shutdown"] = shutdown_status - # Import notification functions from caliper + # Import notification functions from caliper (inside function to avoid circular imports) from projects.caliper.orchestration.notification import ( format_postprocess_status_notification, - parse_postprocess_result, + parse_postprocess_status, ) # Parse postprocess result @@ -827,8 +825,6 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> def _check_job_shutdown_status() -> dict[str, Any] | None: """Check if the job has been aborted via spec.shutdown field.""" try: - from projects.core.library import ci as ci_lib - metadata_dir = ci_lib.get_ci_metadata_dir() fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" if not fournos_fjob_path.exists(): @@ -855,7 +851,6 @@ def _read_step_exit_status( step_dir: Path, current_step_name: str | None = None ) -> tuple[str, StepStatus]: """Read exit status from step directory and return emoji and status enum.""" - from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME try: exit_status_file = step_dir / CI_METADATA_DIRNAME / "exit_status.yaml" @@ -914,8 +909,6 @@ def _check_postprocess_warnings(step_dir: Path) -> StepStatus: def _get_overall_status_from_steps(artifact_dir: Path) -> str: """Check all step exit statuses and return overall status emoji.""" - from projects.core.library import env - from projects.core.library.export import StepStatus try: current_step_name = Path(env.BASE_ARTIFACT_DIR).name @@ -1010,7 +1003,6 @@ def _create_mlflow_step_url(mlflow_run_url: str, step_dir_name: str) -> str | No def _read_step_duration(step_dir: Path) -> str: """Read step duration from timing file.""" - from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME timing_file = step_dir / CI_METADATA_DIRNAME / "test_duration.yaml" if not timing_file.exists(): @@ -1029,7 +1021,6 @@ def _read_step_duration(step_dir: Path) -> str: def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: """Process step logs from parent directory.""" - from projects.core.library import env if not mlflow_run_url: logging.warning("mlflow_run_url not set. Will generate dummy links.") From 9d9574f17ba41c9bb4829f9db36955443959cb49 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:08:10 +0200 Subject: [PATCH 0814/1072] [core] library: export_notifications: add debug logs --- projects/core/library/export_notifications.py | 74 +++++++++++++++++-- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 203a55747..83678eee0 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -205,11 +205,21 @@ def _get_project_and_args(project: str) -> tuple[str, str]: def _extract_finish_reason_from_status(status: dict[str, Any]) -> str: """Extract finish reason from status.""" - if not status.get("success", False): + success = status.get("success", False) + censoring_occurred = status.get("censoring_occurred", False) + + logger.info( + f"DEBUG: Status analysis - success={success}, censoring_occurred={censoring_occurred}" + ) + + if not success: + logger.info(f"DEBUG: Finish reason: failed (success={success})") return "failed" - elif status.get("censoring_occurred", False): + elif censoring_occurred: + logger.info("DEBUG: Finish reason: completed with censoring") return "completed with censoring" else: + logger.info("DEBUG: Finish reason: completed") return "completed" @@ -222,12 +232,25 @@ def _build_enhanced_notification( """Build enhanced notification with fournos job config and artifact links.""" fjob_project, fjob_args_str = _get_project_and_args(project) - status_emoji = "✅" if status.get("success", False) else "❌" - if status.get("censoring_occurred", False): + success = status.get("success", False) + censoring_occurred = status.get("censoring_occurred", False) + + logger.info( + f"DEBUG: Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" + ) + + status_emoji = "✅" if success else "❌" + logger.info(f"DEBUG: Initial emoji based on success: {status_emoji}") + + if censoring_occurred: status_emoji = "⚠️" + logger.info(f"DEBUG: Changed emoji to warning due to censoring: {status_emoji}") if finish_reason == "failed": status_emoji = "❌" + logger.info(f"DEBUG: Changed emoji to failed due to finish_reason: {status_emoji}") + + logger.info(f"DEBUG: Final status emoji: {status_emoji}") base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {status_emoji}" notification_parts = [base_status, ""] @@ -517,13 +540,21 @@ def _get_postprocess_status_links( artifact_dir: Path | None, mlflow_run_url: str | None ) -> list[str]: """Get postprocess status links.""" + logger.info( + f"DEBUG: Postprocess status - artifact_dir={artifact_dir}, mlflow_run_url={mlflow_run_url}" + ) + if not artifact_dir or not artifact_dir.exists(): + logger.info("DEBUG: Postprocess status - artifact_dir missing or doesn't exist") return [] step_log_links = [] # Look for postprocess results in step directories - for step_dir in sorted(artifact_dir.glob("*")): + step_dirs = sorted(artifact_dir.glob("*")) + logger.info(f"DEBUG: Postprocess status - found {len(step_dirs)} directories in {artifact_dir}") + + for step_dir in step_dirs: if not step_dir.is_dir() or step_dir.name.startswith("."): continue @@ -535,15 +566,28 @@ def _get_postprocess_status_links( try: postprocess_status_file = step_dir / "postprocess_status.yaml" + logger.info( + f"DEBUG: Postprocess status - checking {step_name}, file={postprocess_status_file}" + ) + if not postprocess_status_file.exists(): + logger.info( + f"DEBUG: Postprocess status - {step_name}: postprocess_status.yaml not found" + ) continue + logger.info( + f"DEBUG: Postprocess status - {step_name}: postprocess_status.yaml exists, reading..." + ) with open(postprocess_status_file, encoding="utf-8") as f: status_data = yaml.safe_load(f.read()) if not status_data: + logger.info(f"DEBUG: Postprocess status - {step_name}: status_data is empty") continue + logger.info(f"DEBUG: Postprocess status - {step_name}: loaded status_data successfully") + # Add job shutdown status if available if "job_shutdown" in status_data: shutdown_status = status_data["job_shutdown"] @@ -556,10 +600,16 @@ def _get_postprocess_status_links( ) # Parse postprocess result + logger.info(f"DEBUG: Postprocess status - {step_name}: parsing status_data...") result = parse_postprocess_status(status_data) if not result: + logger.info( + f"DEBUG: Postprocess status - {step_name}: parse_postprocess_status returned None/empty" + ) continue + logger.info(f"DEBUG: Postprocess status - {step_name}: parsed result successfully") + # Create file link function for this step def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: if mlflow_run_url: @@ -567,15 +617,29 @@ def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: return _create_mlflow_file_url_for_step( mlflow_run_url, step_subdir, str(file_path) ) + else: + # Fallback: just return the file path as text + logger.info(f"DEBUG: Fallback: just return the file path as text ({file_path})") + + return str(file_path) # Generate notification text from the structured result + logger.info(f"DEBUG: Postprocess status - {step_name}: generating notification text...") notification_text = format_postprocess_status_notification(result, get_file_link) if notification_text: + logger.info( + f"DEBUG: Postprocess status - {step_name}: notification text generated, adding to list" + ) step_log_links.append(notification_text) + else: + logger.info(f"DEBUG: Postprocess status - {step_name}: notification text is empty") except Exception as e: logger.exception(f"Failed to process postprocess status for {step_name}: {e}") + logger.info( + f"DEBUG: Postprocess status - returning {len(step_log_links)} postprocess status links" + ) return step_log_links From 61a7be41c75401a1052defedbd3ba66b31924649 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:10:26 +0200 Subject: [PATCH 0815/1072] [core] library: export_notifications: don't swallow exceptions --- projects/core/library/export_notifications.py | 93 ++++++++++++------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 83678eee0..d85bda3ae 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -269,51 +269,78 @@ def _build_enhanced_notification( notification_parts.append(execution_engine_config) notification_success = True + + # Extract artifact links (with error handling) try: artifact_links, mlflow_run_url = _extract_artifact_links(status) + except Exception as e: + logger.exception(f"Failed to extract artifact links: {e}") + artifact_links, mlflow_run_url = [], None + notification_success = False + # Extract test status section (with error handling) + try: test_status_section = _extract_test_status_section(status) + except Exception as e: + logger.exception(f"Failed to extract test status: {e}") + test_status_section = None + + # Extract step status (with error handling) + try: step_status = _get_step_status_section(artifact_dir, mlflow_run_url) + except Exception as e: + logger.exception(f"Failed to extract step status: {e}") + step_status = None + + # Extract postprocess status (with error handling - don't let this break the notification) + try: postprocess_status_links = _get_postprocess_status_links(artifact_dir, mlflow_run_url) + except Exception as e: + logger.exception(f"Failed to extract postprocess status: {e}") + postprocess_status_links = [] - if test_status_section: - notification_parts.append("") - notification_parts.append("---") - notification_parts.extend(test_status_section) + # Extract censoring report (with error handling) + try: + censoring_report_section = _get_censoring_report_section(artifact_dir) + except Exception as e: + logger.exception(f"Failed to extract censoring report: {e}") + censoring_report_section = None - if artifact_links: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Artifact Links**") - notification_parts.extend([f"* {link}" for link in artifact_links]) - else: - notification_parts.append("**Artifact Links:** No direct links available") + # Build notification sections + if test_status_section: + notification_parts.append("") + notification_parts.append("---") + notification_parts.extend(test_status_section) - if step_status: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Step details**") - for link in step_status: - notification_parts.append(link) + if artifact_links: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Artifact Links**") + notification_parts.extend([f"* {link}" for link in artifact_links]) + else: + if notification_success: + notification_parts.append("**Artifact Links:** No direct links available") + else: + notification_parts.append("**Artifact Links:** Error extracting links") - if postprocess_status_links: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Post-processing Status** ✅") - notification_parts.extend(postprocess_status_links) + if step_status: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Step details**") + for link in step_status: + notification_parts.append(link) - # Add censoring report section if available - censoring_report_section = _get_censoring_report_section(artifact_dir) - if censoring_report_section: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Censoring Report**") - notification_parts.extend(censoring_report_section) + if postprocess_status_links: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Post-processing Status** ✅") + notification_parts.extend(postprocess_status_links) - except Exception as e: - logger.exception(f"Failed to build the extended notifications: {e}") - notification_parts.append("**Artifact Links:** Error extracting links") - notification_success = False + if censoring_report_section: + notification_parts.append("") + notification_parts.append("---") + notification_parts.append("**Censoring Report**") + notification_parts.extend(censoring_report_section) notification_parts.append("") notification_parts.append("---") From 5091962f035e48de4852daea5be718b9f3e503d5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:21:53 +0200 Subject: [PATCH 0816/1072] [caliper] orchestration: postprocess: move the postprocess_status.yaml to the status_files dir --- projects/caliper/orchestration/postprocess.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 1d0bab75c..1ec518c9c 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1523,8 +1523,9 @@ def _save_postprocess_status_yaml(self, result: dict[str, Any]) -> None: save_postprocess_status_yaml, ) - self.output_dir.mkdir(parents=True, exist_ok=True) - status_file = self.output_dir / "postprocess_status.yaml" + output_dir = self.output_dir / "status_files" + output_dir.mkdir(parents=True, exist_ok=True) + status_file = output_dir / "postprocess_status.yaml" # Convert to typed status object status = PostprocessStatus.from_orchestration_result(result) From 0e93efe45fd83dbbab595d6a8ec375b64390abc5 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:22:10 +0200 Subject: [PATCH 0817/1072] [core] library: export_notifications: remove irrelevant code --- projects/core/library/export_notifications.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index d85bda3ae..820cce144 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -615,11 +615,6 @@ def _get_postprocess_status_links( logger.info(f"DEBUG: Postprocess status - {step_name}: loaded status_data successfully") - # Add job shutdown status if available - if "job_shutdown" in status_data: - shutdown_status = status_data["job_shutdown"] - status_data["job_shutdown"] = shutdown_status - # Import notification functions from caliper (inside function to avoid circular imports) from projects.caliper.orchestration.notification import ( format_postprocess_status_notification, From 231a5d635621ed43734329c29cce012e3a81a771 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:30:12 +0200 Subject: [PATCH 0818/1072] [caliper] orchestration: postprocess: define and use POSTPROCESS_STATUS_FILENAME=postprocess_status.yaml --- projects/caliper/orchestration/postprocess.py | 5 +- projects/core/library/export.py | 5 +- projects/core/library/export_notifications.py | 108 ++++++++++-------- projects/core/library/postprocess.py | 11 -- projects/core/library/reports_index.py | 4 +- projects/core/notifications/send.py | 11 +- 6 files changed, 80 insertions(+), 64 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index 1ec518c9c..ac2c7e1a5 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -64,6 +64,9 @@ logger = logging.getLogger(__name__) +# Postprocess status filename constant +POSTPROCESS_STATUS_FILENAME = "postprocess_status.yaml" + def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: """Convert absolute path to relative path from base directory. @@ -1525,7 +1528,7 @@ def _save_postprocess_status_yaml(self, result: dict[str, Any]) -> None: output_dir = self.output_dir / "status_files" output_dir.mkdir(parents=True, exist_ok=True) - status_file = output_dir / "postprocess_status.yaml" + status_file = output_dir / POSTPROCESS_STATUS_FILENAME # Convert to typed status object status = PostprocessStatus.from_orchestration_result(result) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 2f601574b..14262d1a9 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -20,6 +20,7 @@ ExportFailedException, run_from_orchestration_config, ) +from projects.caliper.orchestration.postprocess import POSTPROCESS_STATUS_FILENAME from projects.core.library import ci as ci_lib from projects.core.library import config, env, run from projects.core.library.export_notifications import ( @@ -97,8 +98,8 @@ def _update_fjob_export_status(status: dict): def _process_caliper_postprocess_status( step_dir: Path, step_log_links: list[str], mlflow_run_url: str | None = None ) -> None: - """Search for and process postprocess_status.yaml files in step directory.""" - status_files = list(step_dir.glob("**/postprocess_status.yaml")) + """Search for and process POSTPROCESS_STATUS_FILENAME files in step directory.""" + status_files = list(step_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}")) for status_file in status_files: try: diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 820cce144..60a5a585b 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -14,6 +14,7 @@ import yaml from projects.caliper.orchestration.censoring import censor_text +from projects.caliper.orchestration.postprocess import POSTPROCESS_STATUS_FILENAME from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.library import ci as ci_lib from projects.core.library import config, env @@ -205,6 +206,9 @@ def _get_project_and_args(project: str) -> tuple[str, str]: def _extract_finish_reason_from_status(status: dict[str, Any]) -> str: """Extract finish reason from status.""" + logger.info(f"DEBUG: Raw status object keys: {list(status.keys())}") + logger.info(f"DEBUG: Raw status object: {status}") + success = status.get("success", False) censoring_occurred = status.get("censoring_occurred", False) @@ -592,69 +596,83 @@ def _get_postprocess_status_links( continue try: - postprocess_status_file = step_dir / "postprocess_status.yaml" + # Search for postprocess status files recursively within this step directory + step_postprocess_files = list(step_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}")) logger.info( - f"DEBUG: Postprocess status - checking {step_name}, file={postprocess_status_file}" + f"DEBUG: Postprocess status - {step_name}: found {len(step_postprocess_files)} files: {step_postprocess_files}" ) - if not postprocess_status_file.exists(): + if not step_postprocess_files: logger.info( - f"DEBUG: Postprocess status - {step_name}: postprocess_status.yaml not found" + f"DEBUG: Postprocess status - {step_name}: no {POSTPROCESS_STATUS_FILENAME} files found" ) continue - logger.info( - f"DEBUG: Postprocess status - {step_name}: postprocess_status.yaml exists, reading..." - ) - with open(postprocess_status_file, encoding="utf-8") as f: - status_data = yaml.safe_load(f.read()) - - if not status_data: - logger.info(f"DEBUG: Postprocess status - {step_name}: status_data is empty") - continue - - logger.info(f"DEBUG: Postprocess status - {step_name}: loaded status_data successfully") - # Import notification functions from caliper (inside function to avoid circular imports) from projects.caliper.orchestration.notification import ( format_postprocess_status_notification, parse_postprocess_status, ) - # Parse postprocess result - logger.info(f"DEBUG: Postprocess status - {step_name}: parsing status_data...") - result = parse_postprocess_status(status_data) - if not result: + # Process ALL postprocess status files found + for file_index, postprocess_status_file in enumerate(step_postprocess_files): logger.info( - f"DEBUG: Postprocess status - {step_name}: parse_postprocess_status returned None/empty" + f"DEBUG: Postprocess status - {step_name}: processing file {file_index + 1}/{len(step_postprocess_files)}: {postprocess_status_file}" ) - continue - logger.info(f"DEBUG: Postprocess status - {step_name}: parsed result successfully") + with open(postprocess_status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f.read()) - # Create file link function for this step - def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: - if mlflow_run_url: - # Create MLflow artifact URL - return _create_mlflow_file_url_for_step( - mlflow_run_url, step_subdir, str(file_path) + if not status_data: + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} status_data is empty, skipping" ) - else: - # Fallback: just return the file path as text - logger.info(f"DEBUG: Fallback: just return the file path as text ({file_path})") + continue - return str(file_path) + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} loaded status_data successfully" + ) - # Generate notification text from the structured result - logger.info(f"DEBUG: Postprocess status - {step_name}: generating notification text...") - notification_text = format_postprocess_status_notification(result, get_file_link) - if notification_text: + # Parse postprocess result logger.info( - f"DEBUG: Postprocess status - {step_name}: notification text generated, adding to list" + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parsing status_data..." ) - step_log_links.append(notification_text) - else: - logger.info(f"DEBUG: Postprocess status - {step_name}: notification text is empty") + result = parse_postprocess_status(status_data) + if not result: + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parse_postprocess_status returned None/empty" + ) + continue + + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parsed result successfully" + ) + + # Create file link function for this step + def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: + if mlflow_run_url: + # Create MLflow artifact URL + return _create_mlflow_file_url_for_step( + mlflow_run_url, step_subdir, str(file_path) + ) + else: + # Fallback: just return the file path as text + return str(file_path) + + # Generate notification text from the structured result + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} generating notification text..." + ) + notification_text = format_postprocess_status_notification(result, get_file_link) + if notification_text: + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} notification text generated, adding to list" + ) + step_log_links.append(notification_text) + else: + logger.info( + f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} notification text is empty" + ) except Exception as e: logger.exception(f"Failed to process postprocess status for {step_name}: {e}") @@ -831,15 +849,15 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: - """Extract post-processing status information from postprocess_status.yaml files. + """Extract post-processing status information from POSTPROCESS_STATUS_FILENAME files. Returns: List of formatted strings with postprocess step status (success only, no details) """ postprocess_info_lines = [] - # Search for postprocess_status.yaml files recursively - postprocess_files = list(artifact_dir.glob("**/postprocess_status.yaml")) + # Search for POSTPROCESS_STATUS_FILENAME files recursively + postprocess_files = list(artifact_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}")) if not postprocess_files: return [] @@ -966,7 +984,7 @@ def _check_postprocess_warnings(step_dir: Path) -> StepStatus: """Check for warning status in postprocess status file.""" status = StepStatus.SUCCESS # No postprocess warning/error, assume no warnings - for status_file in step_dir.glob("**/postprocess_status.yaml"): + for status_file in step_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}"): try: with open(status_file, encoding="utf-8") as f: status_data = yaml.safe_load(f) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index bd5a16331..60e61a27e 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -411,17 +411,6 @@ def run_orchestration_postprocess( test_outcome=test_outcome, ) - status_path = output_dir / "postprocess_status.yaml" - try: - status_path.parent.mkdir(parents=True, exist_ok=True) - status_path.write_text( - yaml.dump(result, indent=2, default_flow_style=False, sort_keys=False), - encoding="utf-8", - ) - logger.info("Wrote postprocess status YAML to %s", status_path) - except OSError as e: - logger.warning("Could not write %s: %s", status_path, e) - # Generate HTML reports try: generate_caliper_reports_index(result, output_dir or env.ARTIFACT_DIR, "reports_index.html") diff --git a/projects/core/library/reports_index.py b/projects/core/library/reports_index.py index ef650a59e..d7f775102 100644 --- a/projects/core/library/reports_index.py +++ b/projects/core/library/reports_index.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Any +from projects.caliper.orchestration.postprocess import POSTPROCESS_STATUS_FILENAME + logger = logging.getLogger(__name__) @@ -52,7 +54,7 @@ def generate_caliper_reports_index( html_files = [] data_files = [] - ignored_files = {index_filename, "postprocess_status.yaml"} + ignored_files = {index_filename, POSTPROCESS_STATUS_FILENAME} for html_file in sorted(output_dir.glob("*.html")): if html_file.name not in ignored_files: diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 04a9b8f61..267e9847f 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -20,6 +20,9 @@ DEFAULT_REPO_OWNER = "openshift-psap" DEFAULT_REPO_NAME = "forge" +# avoid importing projects.caliper.orchestration.postprocess here +POSTPROCESS_STATUS_FILENAME = "postprocess_status.yaml" + def get_secrets(notification_vault=None): """Get secrets directory, preferring vault system over environment variables. @@ -345,8 +348,8 @@ def get_common_message(finish_reason: str, status: str, get_link, get_italics, g artifact_dir = pathlib.Path(os.environ.get("ARTIFACT_DIR", "")) caliper_status_path = None - # Search for postprocess_status.yaml in artifact directory and subdirectories - for status_file in artifact_dir.glob("**/postprocess_status.yaml"): + # Search for POSTPROCESS_STATUS_FILENAME in artifact directory and subdirectories + for status_file in artifact_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}"): caliper_status_path = status_file break @@ -376,9 +379,9 @@ def get_common_message(finish_reason: str, status: str, get_link, get_italics, g • Caliper postprocess completed but no reports generated. """ except Exception as e: - logger.warning("Failed to parse postprocess_status.yaml: %s", e) + logger.warning("Failed to parse POSTPROCESS_STATUS_FILENAME: %s", e) message += """ -• Failed to parse postprocess_status.yaml ... +• Failed to parse POSTPROCESS_STATUS_FILENAME ... """ # Include fournos_launcher generated notification content From 781165171d63593a07c19cd3253c7e52f289b5f8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 14:58:26 +0200 Subject: [PATCH 0819/1072] [core] library: export_notifications: reduce the verbosity --- projects/core/library/export_notifications.py | 47 ++++++------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 60a5a585b..d9938528b 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -240,19 +240,19 @@ def _build_enhanced_notification( censoring_occurred = status.get("censoring_occurred", False) logger.info( - f"DEBUG: Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" + f"Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" ) status_emoji = "✅" if success else "❌" - logger.info(f"DEBUG: Initial emoji based on success: {status_emoji}") + logger.info(f"Initial emoji based on success: {status_emoji}") if censoring_occurred: status_emoji = "⚠️" - logger.info(f"DEBUG: Changed emoji to warning due to censoring: {status_emoji}") + logger.info(f"Changed emoji to warning due to censoring: {status_emoji}") if finish_reason == "failed": status_emoji = "❌" - logger.info(f"DEBUG: Changed emoji to failed due to finish_reason: {status_emoji}") + logger.info(f"Changed emoji to failed due to finish_reason: {status_emoji}") logger.info(f"DEBUG: Final status emoji: {status_emoji}") @@ -337,7 +337,6 @@ def _build_enhanced_notification( if postprocess_status_links: notification_parts.append("") notification_parts.append("---") - notification_parts.append("**Post-processing Status** ✅") notification_parts.extend(postprocess_status_links) if censoring_report_section: @@ -598,13 +597,10 @@ def _get_postprocess_status_links( try: # Search for postprocess status files recursively within this step directory step_postprocess_files = list(step_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}")) - logger.info( - f"DEBUG: Postprocess status - {step_name}: found {len(step_postprocess_files)} files: {step_postprocess_files}" - ) if not step_postprocess_files: logger.info( - f"DEBUG: Postprocess status - {step_name}: no {POSTPROCESS_STATUS_FILENAME} files found" + f"Postprocess status - {step_name}: no {POSTPROCESS_STATUS_FILENAME} files found" ) continue @@ -615,37 +611,25 @@ def _get_postprocess_status_links( ) # Process ALL postprocess status files found - for file_index, postprocess_status_file in enumerate(step_postprocess_files): - logger.info( - f"DEBUG: Postprocess status - {step_name}: processing file {file_index + 1}/{len(step_postprocess_files)}: {postprocess_status_file}" - ) - + for postprocess_status_file in step_postprocess_files: with open(postprocess_status_file, encoding="utf-8") as f: status_data = yaml.safe_load(f.read()) if not status_data: - logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} status_data is empty, skipping" + logger.warning( + f"Postprocess status - {postprocess_status_file} status_data is empty, skipping" ) continue - logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} loaded status_data successfully" - ) - - # Parse postprocess result - logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parsing status_data..." - ) result = parse_postprocess_status(status_data) if not result: - logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parse_postprocess_status returned None/empty" + logger.warning( + f"Postprocess status - {postprocess_status_file} parse_postprocess_status returned None/empty" ) continue logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} parsed result successfully" + f"Postprocess status - {postprocess_status_file} parsed result successfully" ) # Create file link function for this step @@ -661,25 +645,22 @@ def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: # Generate notification text from the structured result logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} generating notification text..." + f"Postprocess status - {postprocess_status_file} generating notification text..." ) notification_text = format_postprocess_status_notification(result, get_file_link) if notification_text: logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} notification text generated, adding to list" + f"Postprocess status - {postprocess_status_file} notification text generated, adding to list" ) step_log_links.append(notification_text) else: logger.info( - f"DEBUG: Postprocess status - {step_name}: file {file_index + 1} notification text is empty" + f"Postprocess status - {postprocess_status_file} notification text is empty" ) except Exception as e: logger.exception(f"Failed to process postprocess status for {step_name}: {e}") - logger.info( - f"DEBUG: Postprocess status - returning {len(step_log_links)} postprocess status links" - ) return step_log_links From 1873f95a05304a7a917e30d23454ab0e4bfefe42 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 15:41:34 +0200 Subject: [PATCH 0820/1072] [caliper] orchestration: export: use dataclasses for the status --- projects/caliper/orchestration/export.py | 10 +- projects/core/library/export.py | 76 +++-- projects/core/library/export_notifications.py | 322 ++++++++++++++---- 3 files changed, 313 insertions(+), 95 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 696ed8f02..87bffc8d7 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -36,6 +36,7 @@ from projects.core.library import env from projects.core.library import vault as vault_lib from projects.core.library.config import requires +from projects.core.library.export_notifications import ExportStatus logger = logging.getLogger(__name__) @@ -169,7 +170,7 @@ def run_from_orchestration_config( caliper_cfg: dict[str, Any] | None, disable_censoring: bool = False, disable_file_export: bool = False, -) -> int: +) -> ExportStatus: """ Run Caliper file export from orchestration config. @@ -318,11 +319,14 @@ def run_from_orchestration_config( raise ExportFailedException(f"Artifacts export failed (ret code = {ret})") with open(status_yaml) as f: - status = yaml.safe_load(f.read()) + status_dict = yaml.safe_load(f.read()) + + # Create ExportStatus from loaded data + status = ExportStatus.from_dict(status_dict) # Add censoring information to status if len(run_dirs) == 1: - status["censoring_occurred"] = censoring_occurred + status.censoring_occurred = censoring_occurred return status diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 14262d1a9..3e8cf0332 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -24,6 +24,10 @@ from projects.core.library import ci as ci_lib from projects.core.library import config, env, run from projects.core.library.export_notifications import ( + BackendResult, + CaliperArtifactsExport, + ExportStatus, + TestPhase, _check_job_shutdown_status, _create_mlflow_file_url_for_step, send_notification, @@ -38,7 +42,7 @@ class FinishReason(StrEnum): OTHER = "other" -def _update_fjob_export_status(status: dict): +def _update_fjob_export_status(status: ExportStatus): """Update FournosJob status with export artifacts status.""" if not env.running_inside_fournos(): return @@ -75,7 +79,7 @@ def _update_fjob_export_status(status: dict): fjob_data["status"]["engineStatus"]["forge"]["status"] = {} # Update with export-artifacts status - fjob_data["status"]["engineStatus"]["forge"]["exportArtifacts"] = status + fjob_data["status"]["engineStatus"]["forge"]["exportArtifacts"] = status.to_dict() # Patch the fjob patch_data = {"status": fjob_data["status"]} @@ -297,34 +301,35 @@ def caliper_export_entrypoint( "DRY RUN: Skipping actual caliper export, creating mock status for notification" ) # Create a realistic mock status for notification testing - status = { - "success": True, - "final_status": "success", - "backends": {}, - "caliper_artifacts_export": { - "backends": { - "mlflow": { - "success": True, - "run_id": "dry-run-mock-id", - "experiment_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123", - "run_url": "http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", - "tracking_uri": "http://DRY_RUN_MLFLOW_FAKE_URL", - } - } - }, - "duration": "15 minutes, 30 seconds", - "test_phase": { - "phase": "FAILED", - "message": "Test execution completed with failures", - }, - } + status = ExportStatus( + success=True, + final_status="success", + censoring_occurred=False, + duration="15 minutes, 30 seconds", + caliper_artifacts_export=CaliperArtifactsExport( + version=1, + backends={ + "mlflow": BackendResult( + success=True, + run_id="dry-run-mock-id", + experiment_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123", + run_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", + tracking_uri="http://DRY_RUN_MLFLOW_FAKE_URL", + ) + }, + ), + test_phase=TestPhase( + phase="FAILED", + message="Test execution completed with failures", + ), + ) else: status = run_caliper_orchestration_export( artifact_dir=artifact_dir, disable_censoring=disable_censoring, disable_file_export=disable_file_export, ) - logger.info("Export status:\n" + yaml.dump(status, indent=4)) + logger.info("Export status:\n" + yaml.dump(status.to_dict(), indent=4)) # Update fjob status with export results (only if file export is not disabled) if not disable_file_export: @@ -336,12 +341,12 @@ def caliper_export_entrypoint( logger.exception(f"Export failed: {e}") export_failed = True # Create failure status for notification - status = {"success": False, "error": str(e), "backends": {}} + status = ExportStatus(success=False, final_status=f"failed: {e}") except Exception as e: logger.exception(f"Export failed with unexpected error: {e}") export_failed = True # Create failure status for notification - status = {"success": False, "error": str(e), "backends": {}} + status = ExportStatus(success=False, final_status=f"failed: {e}") finally: # Send completion notifications regardless of success/failure @@ -374,22 +379,31 @@ def caliper_export_entrypoint( return 1, "failed" # Check if censoring occurred and return exit code 1 if so - if status and status.get("censoring_occurred", False): + if status and status.censoring_occurred: return 1, "censoring_occurred" return 0 -def _update_final_artifacts(artifact_dir, export_status: dict[str, Any] | None) -> None: +def _update_final_artifacts(artifact_dir, export_status: ExportStatus | None) -> None: """Update the final artifacts (run.log, notifications) to MLflow after all post-export work is done.""" if not export_status: logger.warning("No export status received, cannot update the final artifacts") return try: - caliper_export = export_status.get("caliper_artifacts_export", {}) - backends = caliper_export.get("backends", {}) - mlflow_meta = backends.get("mlflow") + caliper_export = export_status.caliper_artifacts_export + if not caliper_export or not caliper_export.backends: + logger.warning( + "Export status doesn't have caliper artifacts export data, cannot update the final artifacts" + ) + return + + mlflow_backend = caliper_export.backends.get("mlflow") + if isinstance(mlflow_backend, BackendResult): + mlflow_meta = mlflow_backend.to_dict() + else: + mlflow_meta = mlflow_backend if not isinstance(mlflow_meta, dict): logger.warning( "Export status don't have the mlflow backend, cannot update the final artifacts" diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index d9938528b..9b3ff8c93 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -8,6 +8,7 @@ import logging import os import re +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -25,6 +26,202 @@ logger = logging.getLogger(__name__) +@dataclass +class BackendResult: + """Result from a single backend export.""" + + success: bool = False + run_id: str | None = None + experiment_url: str | None = None + run_url: str | None = None + tracking_uri: str | None = None + detail: str | None = None + status: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "BackendResult": + """Create BackendResult from raw dict.""" + return cls( + success=data.get("success", False), + run_id=data.get("run_id"), + experiment_url=data.get("experiment_url"), + run_url=data.get("run_url"), + tracking_uri=data.get("tracking_uri"), + detail=data.get("detail"), + status=data.get("status"), + ) + + def to_dict(self) -> dict[str, Any]: + """Convert to dict.""" + result = {"success": self.success} + if self.run_id is not None: + result["run_id"] = self.run_id + if self.experiment_url is not None: + result["experiment_url"] = self.experiment_url + if self.run_url is not None: + result["run_url"] = self.run_url + if self.tracking_uri is not None: + result["tracking_uri"] = self.tracking_uri + if self.detail is not None: + result["detail"] = self.detail + if self.status is not None: + result["status"] = self.status + return result + + +@dataclass +class CaliperArtifactsExport: + """Caliper artifacts export information.""" + + version: int = 1 + backends: dict[str, BackendResult] | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "CaliperArtifactsExport": + """Create CaliperArtifactsExport from raw dict.""" + backends_data = data.get("backends", {}) + backends = {} + for backend_name, backend_data in backends_data.items(): + if isinstance(backend_data, dict): + backends[backend_name] = BackendResult.from_dict(backend_data) + else: + backends[backend_name] = backend_data # Keep non-dict values as-is + + return cls( + version=data.get("version", 1), + backends=backends if backends else None, + ) + + def to_dict(self) -> dict[str, Any]: + """Convert to dict.""" + result = {"version": self.version} + if self.backends: + backends_dict = {} + for backend_name, backend_result in self.backends.items(): + if isinstance(backend_result, BackendResult): + backends_dict[backend_name] = backend_result.to_dict() + else: + backends_dict[backend_name] = backend_result + result["backends"] = backends_dict + return result + + +@dataclass +class TestPhase: + """Test execution phase information.""" + + phase: str + message: str = "" + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "TestPhase": + """Create TestPhase from raw dict.""" + return cls( + phase=data.get("phase", "UNKNOWN"), + message=data.get("message", ""), + ) + + def to_dict(self) -> dict[str, Any]: + """Convert to dict.""" + return { + "phase": self.phase, + "message": self.message, + } + + +@dataclass +class JobShutdown: + """Job shutdown/abort information.""" + + is_aborted: bool = False + shutdown_value: str | None = None + shutdown_detected: bool = False + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "JobShutdown": + """Create JobShutdown from raw dict.""" + return cls( + is_aborted=data.get("is_aborted", False), + shutdown_value=data.get("shutdown_value"), + shutdown_detected=data.get("shutdown_detected", False), + ) + + def to_dict(self) -> dict[str, Any]: + """Convert to dict.""" + result = { + "is_aborted": self.is_aborted, + "shutdown_detected": self.shutdown_detected, + } + if self.shutdown_value is not None: + result["shutdown_value"] = self.shutdown_value + return result + + +@dataclass +class ExportStatus: + """Dataclass for caliper export status.""" + + success: bool + final_status: str + censoring_occurred: bool = False + duration: str | None = None + caliper_artifacts_export: CaliperArtifactsExport | None = None + test_phase: TestPhase | None = None + job_shutdown: JobShutdown | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ExportStatus": + """Create ExportStatus from raw dict, providing proper defaults.""" + # Extract success from various possible sources + success = data.get("success") + if success is None: + # Try to determine success from final_status + final_status = data.get("final_status", "") + success = final_status in ("success", "completed") + + # Convert nested structures to typed objects + caliper_export = None + if data.get("caliper_artifacts_export"): + caliper_export = CaliperArtifactsExport.from_dict(data["caliper_artifacts_export"]) + + test_phase = None + if data.get("test_phase"): + test_phase = TestPhase.from_dict(data["test_phase"]) + + job_shutdown = None + if data.get("job_shutdown"): + job_shutdown = JobShutdown.from_dict(data["job_shutdown"]) + + return cls( + success=success, + final_status=data.get("final_status", "unknown"), + censoring_occurred=data.get("censoring_occurred", False), + duration=data.get("duration"), + caliper_artifacts_export=caliper_export, + test_phase=test_phase, + job_shutdown=job_shutdown, + ) + + def to_dict(self) -> dict[str, Any]: + """Convert back to dict for compatibility.""" + result = { + "success": self.success, + "final_status": self.final_status, + "censoring_occurred": self.censoring_occurred, + } + + if self.duration is not None: + result["duration"] = self.duration + if self.caliper_artifacts_export is not None: + result["caliper_artifacts_export"] = self.caliper_artifacts_export.to_dict() + if self.test_phase is not None: + result["test_phase"] = self.test_phase.to_dict() + if self.job_shutdown is not None: + result["job_shutdown"] = self.job_shutdown.to_dict() + + return result + + def _censor_notification_text(text: str, verbose: bool = False) -> str: """ Censor sensitive content in notification text using caliper orchestration. @@ -50,7 +247,7 @@ def _censor_notification_text(text: str, verbose: bool = False) -> str: def send_notification( artifact_dir: Path | None, - status: dict[str, Any], + status: ExportStatus, notification_provider=None, dry_run: bool = False, ) -> bool: @@ -58,7 +255,7 @@ def send_notification( Args: artifact_dir: Directory to browse to find the artifacts - status: Caliper export status object containing backend results and metadata + status: Caliper export status dataclass notification_provider: Optional per-project SlackNotificationProvider instance dry_run: If True, only build and log notification content without sending @@ -139,7 +336,7 @@ def send_notification( # Censor status fields before creating NotificationContext try: censored_status = yaml.safe_load( - _censor_notification_text(yaml.dump(status), verbose=dry_run) + _censor_notification_text(yaml.dump(status.to_dict()), verbose=dry_run) ) except Exception as e: logger.error( @@ -204,22 +401,18 @@ def _get_project_and_args(project: str) -> tuple[str, str]: return fjob_project, fjob_args_str -def _extract_finish_reason_from_status(status: dict[str, Any]) -> str: +def _extract_finish_reason_from_status(status: ExportStatus) -> str: """Extract finish reason from status.""" - logger.info(f"DEBUG: Raw status object keys: {list(status.keys())}") - logger.info(f"DEBUG: Raw status object: {status}") - - success = status.get("success", False) - censoring_occurred = status.get("censoring_occurred", False) + logger.info(f"DEBUG: Export status object: {status}") logger.info( - f"DEBUG: Status analysis - success={success}, censoring_occurred={censoring_occurred}" + f"DEBUG: Status analysis - success={status.success}, censoring_occurred={status.censoring_occurred}" ) - if not success: - logger.info(f"DEBUG: Finish reason: failed (success={success})") + if not status.success: + logger.info(f"DEBUG: Finish reason: failed (success={status.success})") return "failed" - elif censoring_occurred: + elif status.censoring_occurred: logger.info("DEBUG: Finish reason: completed with censoring") return "completed with censoring" else: @@ -231,13 +424,13 @@ def _build_enhanced_notification( artifact_dir: Path, project: str, finish_reason: str, - status: dict[str, Any], + status: ExportStatus, ) -> tuple[str, bool]: """Build enhanced notification with fournos job config and artifact links.""" fjob_project, fjob_args_str = _get_project_and_args(project) - success = status.get("success", False) - censoring_occurred = status.get("censoring_occurred", False) + success = status.success + censoring_occurred = status.censoring_occurred logger.info( f"Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" @@ -257,18 +450,18 @@ def _build_enhanced_notification( logger.info(f"DEBUG: Final status emoji: {status_emoji}") base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {status_emoji}" - notification_parts = [base_status, ""] + notification_parts = [base_status, ] # Add job abort message right below overall status if applicable - shutdown_status = status.get("job_shutdown") - if shutdown_status and shutdown_status.get("is_aborted"): - notification_parts.append("---") - shutdown_value = shutdown_status.get("shutdown_value", "Stop") + shutdown_status = status.job_shutdown + if shutdown_status and shutdown_status.is_aborted: + notification_parts += ["", "---"] + shutdown_value = shutdown_status.shutdown_value or "Stop" notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") - notification_parts += ["", "---", ""] execution_engine_config = _get_execution_engine_config() if execution_engine_config: + notification_parts += ["", "---"] notification_parts.append("**Execution Engine Configuration**") notification_parts.append(execution_engine_config) @@ -312,13 +505,11 @@ def _build_enhanced_notification( # Build notification sections if test_status_section: - notification_parts.append("") - notification_parts.append("---") + notification_parts += ["", "---"] notification_parts.extend(test_status_section) if artifact_links: - notification_parts.append("") - notification_parts.append("---") + notification_parts += ["", "---"] notification_parts.append("**Artifact Links**") notification_parts.extend([f"* {link}" for link in artifact_links]) else: @@ -328,26 +519,20 @@ def _build_enhanced_notification( notification_parts.append("**Artifact Links:** Error extracting links") if step_status: - notification_parts.append("") - notification_parts.append("---") - notification_parts.append("**Step details**") + notification_parts += ["", "---"] + notification_parts.append("**Pipeline Step Details**") for link in step_status: notification_parts.append(link) if postprocess_status_links: - notification_parts.append("") - notification_parts.append("---") + notification_parts += ["", "---"] notification_parts.extend(postprocess_status_links) if censoring_report_section: - notification_parts.append("") - notification_parts.append("---") + notification_parts += ["", "---"] notification_parts.append("**Censoring Report**") notification_parts.extend(censoring_report_section) - notification_parts.append("") - notification_parts.append("---") - return "\n".join(notification_parts), notification_success @@ -441,14 +626,14 @@ def _get_execution_engine_config() -> str | None: return None -def _extract_test_status_section(status: dict[str, Any]) -> list[str] | None: +def _extract_test_status_section(status: ExportStatus) -> list[str] | None: """Extract test status section from status.""" - test_phase = status.get("test_phase") + test_phase = status.test_phase if not test_phase: return None - phase = test_phase.get("phase", "").upper() - message = test_phase.get("message", "") + phase = test_phase.phase.upper() + message = test_phase.message test_status_emoji = "✅" if phase == "PASSED" else "❌" if phase == "FAILED" else "⚠️" return [ @@ -664,33 +849,48 @@ def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: return step_log_links -def _extract_artifact_links(status: dict[str, Any]) -> tuple[list[str], str | None]: +def _extract_artifact_links(status: ExportStatus) -> tuple[list[str], str | None]: """Extract artifact links and MLflow URL from status.""" artifact_links = [] mlflow_run_url = None - caliper_export = status.get("caliper_artifacts_export", {}) - backends = caliper_export.get("backends", {}) + caliper_export = status.caliper_artifacts_export + if not caliper_export or not caliper_export.backends: + return artifact_links, mlflow_run_url - for backend_name, backend_result in backends.items(): - if not isinstance(backend_result, dict): - continue + for backend_name, backend_result in caliper_export.backends.items(): + if isinstance(backend_result, BackendResult): + # Use typed access for BackendResult objects + if backend_result.experiment_url: + artifact_links.append( + f"[{backend_name} Experiment]({backend_result.experiment_url})" + ) - if backend_result.get("experiment_url"): - artifact_links.append( - f"[{backend_name} Experiment]({backend_result['experiment_url']})" - ) + if backend_result.run_url: + mlflow_run_url = backend_result.run_url + artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") - if backend_result.get("run_url"): - mlflow_run_url = backend_result["run_url"] - artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") - elif backend_result.get("artifact_url"): - artifact_links.append(f"[{backend_name} Artifacts]({backend_result['artifact_url']})") - elif backend_result.get("dashboard_url"): - artifact_links.append(f"[{backend_name} Dashboard]({backend_result['dashboard_url']})") + elif isinstance(backend_result, dict): + # Legacy support for dict-based backend results + if backend_result.get("experiment_url"): + artifact_links.append( + f"[{backend_name} Experiment]({backend_result['experiment_url']})" + ) + + if backend_result.get("run_url"): + mlflow_run_url = backend_result["run_url"] + artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") + elif backend_result.get("artifact_url"): + artifact_links.append( + f"[{backend_name} Artifacts]({backend_result['artifact_url']})" + ) + elif backend_result.get("dashboard_url"): + artifact_links.append( + f"[{backend_name} Dashboard]({backend_result['dashboard_url']})" + ) - if status.get("artifact_url"): - artifact_links.append(f"[Artifacts]({status['artifact_url']})") + # Note: artifact_url is not part of the main export status + # This appears to be legacy code - removing for now return artifact_links, mlflow_run_url @@ -1159,10 +1359,10 @@ def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: return step_status -def _extract_duration_from_status(status: dict[str, Any]) -> str: +def _extract_duration_from_status(status: ExportStatus) -> str: """Extract duration from status object.""" # Look for duration in status - duration = status.get("duration") + duration = status.duration if duration: return f" after {duration}" return "" From 4b7326c595561cc4e851220b1356e21def4e5d67 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 16:00:11 +0200 Subject: [PATCH 0821/1072] [core] library: export_notifications: repopulate the executionEngine field --- projects/core/library/export_notifications.py | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 9b3ff8c93..e9bb44270 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -450,7 +450,7 @@ def _build_enhanced_notification( logger.info(f"DEBUG: Final status emoji: {status_emoji}") base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {status_emoji}" - notification_parts = [base_status, ] + notification_parts = [base_status] # Add job abort message right below overall status if applicable shutdown_status = status.job_shutdown @@ -608,22 +608,32 @@ def _get_censoring_report_section(artifact_dir: Path) -> list[str] | None: def _get_execution_engine_config() -> str | None: - """Get execution engine configuration for notification.""" + """Read and format execution engine configuration.""" try: - cluster_config = config.project.get_config("cluster", None, warn=False) - if not cluster_config: - return None + metadata_dir = ci_lib.get_ci_metadata_dir() + fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" + if not fournos_fjob_path.exists(): + return f"* FournosJob not found at `{fournos_fjob_path}`" - config_parts = [] - for key, value in cluster_config.items(): - if value: - config_parts.append(f"`{key}`: {value}") + with open(fournos_fjob_path, encoding="utf-8") as f: + fjob_data = yaml.safe_load(f) - if config_parts: - return "* " + " \n* ".join(config_parts) - except Exception: - pass - return None + execution_engine = fjob_data.get("spec", {}).get("executionEngine", {}) + + config = {} + config["executionEngine"] = execution_engine + config["owner"] = fjob_data.get("spec", {}).get("owner") + + if cluster := fjob_data.get("spec", {}).get("cluster"): + config["cluster"] = cluster + + if pipeline := fjob_data.get("spec", {}).get("pipeline"): + config["pipeline"] = pipeline + + config_yaml = yaml.dump(config, default_flow_style=False, sort_keys=True) + return f"```yaml\n{config_yaml.strip()}\n```" + except Exception as e: + return f"* Failed to read fournos job config: `{e}`" def _extract_test_status_section(status: ExportStatus) -> list[str] | None: From 1b38b07927325d32b93d127fe76de2b45144237d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 16:45:11 +0200 Subject: [PATCH 0822/1072] [core] library: export_notifications: include the test directory listing --- .../caliper/orchestration/notification.py | 6 +- projects/core/library/export_notifications.py | 246 +++++++++++++++++- 2 files changed, 238 insertions(+), 14 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 2b089d79c..c7bfdebca 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -8,6 +8,8 @@ from __future__ import annotations +from pathlib import Path + from projects.caliper.public import PostprocessStatus, StepStatus @@ -32,7 +34,7 @@ def format_postprocess_status_notification( # Check overall status (keep unchanged regardless of abort status) status_emoji = "✅" if status.is_success() else "❌" - base_directory = status.base_directory + base_directory = Path(status.base_directory) if base_directory.name == "status_files": base_directory = base_directory.parent @@ -341,7 +343,6 @@ def _group_files_by_type(file_paths: list[str]) -> dict[str, list[str]]: def _get_file_type(file_path: str) -> str: """Determine file type from path.""" - from pathlib import Path ext = Path(file_path).suffix.lower() @@ -361,7 +362,6 @@ def _get_file_type(file_path: str) -> str: def _get_display_name(file_path: str) -> str: """Get display name for a file path.""" - from pathlib import Path path = Path(file_path) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index e9bb44270..9b844197f 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -14,6 +14,8 @@ import yaml +from projects.caliper.engine.constants import METADATA_FILE +from projects.caliper.engine.kpi.dataclasses import CaliperTestMetadata from projects.caliper.orchestration.censoring import censor_text from projects.caliper.orchestration.postprocess import POSTPROCESS_STATUS_FILENAME from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME @@ -41,8 +43,14 @@ class BackendResult: @classmethod def from_dict(cls, data: dict[str, Any]) -> "BackendResult": """Create BackendResult from raw dict.""" + # Infer success from status if not explicitly provided + success = data.get("success") + if success is None: + status = data.get("status", "") + success = status == "success" + return cls( - success=data.get("success", False), + success=success, run_id=data.get("run_id"), experiment_url=data.get("experiment_url"), run_url=data.get("run_url"), @@ -172,14 +180,7 @@ class ExportStatus: @classmethod def from_dict(cls, data: dict[str, Any]) -> "ExportStatus": """Create ExportStatus from raw dict, providing proper defaults.""" - # Extract success from various possible sources - success = data.get("success") - if success is None: - # Try to determine success from final_status - final_status = data.get("final_status", "") - success = final_status in ("success", "completed") - - # Convert nested structures to typed objects + # Convert nested structures to typed objects first caliper_export = None if data.get("caliper_artifacts_export"): caliper_export = CaliperArtifactsExport.from_dict(data["caliper_artifacts_export"]) @@ -192,9 +193,41 @@ def from_dict(cls, data: dict[str, Any]) -> "ExportStatus": if data.get("job_shutdown"): job_shutdown = JobShutdown.from_dict(data["job_shutdown"]) + # Extract success from various possible sources + success = data.get("success") + final_status = data.get("final_status", "") + + if success is None: + # Try to determine success from final_status first + if final_status in ("success", "completed"): + success = True + elif final_status in ("failed", "failure", "error"): + success = False + elif caliper_export and caliper_export.backends: + # Infer success from backend results if no explicit status + backend_successes = [] + for backend_result in caliper_export.backends.values(): + if isinstance(backend_result, BackendResult): + backend_successes.append(backend_result.success) + elif isinstance(backend_result, dict): + # Handle raw dict backends (shouldn't happen after conversion) + backend_status = backend_result.get("status", "") + backend_successes.append(backend_status == "success") + + # Overall success if all backends succeeded + success = all(backend_successes) if backend_successes else False + + # Set final_status based on inferred success if it wasn't set + if not final_status or final_status == "unknown": + final_status = "success" if success else "failed" + else: + # Default to failed if we can't determine success + success = False + final_status = final_status or "unknown" + return cls( success=success, - final_status=data.get("final_status", "unknown"), + final_status=final_status, censoring_occurred=data.get("censoring_occurred", False), duration=data.get("duration"), caliper_artifacts_export=caliper_export, @@ -1095,7 +1128,7 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> list[str]: - """Process test labels and postprocess status for a single step directory.""" + """Process test labels, caliper metadata, and postprocess status for a single step directory.""" step_details = [] # Extract test labels for this specific step @@ -1106,6 +1139,28 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> except Exception as e: logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") + # Extract caliper metadata for this specific step (before postprocess status) + try: + # Create file link function for this step + def get_file_link(file_path: Path) -> str: + if mlflow_run_url: + # Create MLflow artifact URL + return _create_mlflow_file_url_for_step( + mlflow_run_url, step_dir.name, str(file_path) + ) + else: + # Fallback: just return the file path as text + return str(file_path) + + metadata_files = _search_caliper_metadata_files(step_dir) + metadata_info = _format_caliper_metadata_info_for_step( + metadata_files, get_file_link, step_dir.parent if step_dir.parent else step_dir + ) + if metadata_info: + step_details.extend(metadata_info) + except Exception as e: + logger.warning(f"Failed to extract caliper metadata for step {step_dir.name}: {e}") + # Extract postprocess status for this specific step try: postprocess_info = _extract_postprocess_status_info(step_dir) @@ -1376,3 +1431,172 @@ def _extract_duration_from_status(status: ExportStatus) -> str: if duration: return f" after {duration}" return "" + + +def _search_caliper_metadata_files(step_dir: Path) -> list[Path]: + """Search for caliper metadata files in the step directory.""" + + if not step_dir.exists(): + return [] + + metadata_files = [] + # Search for METADATA_FILE (__caliper_test_metadata__.yaml) recursively + for metadata_file in step_dir.rglob(METADATA_FILE): + metadata_files.append(metadata_file) + + return metadata_files + + +def _format_caliper_metadata_info(metadata_files: list[Path], get_file_link: Any) -> str: + """Format caliper metadata information for display.""" + if not metadata_files: + return "" + + metadata_lines = [] + metadata_lines.append("**Test Directories**") + + for metadata_file in metadata_files: + try: + # Load and parse metadata + with open(metadata_file, encoding="utf-8") as f: + metadata_dict = yaml.safe_load(f) + + metadata = CaliperTestMetadata.from_dict(metadata_dict) + + # Get parent directory path + parent_dir = metadata_file.parent + + # Extract information + labels = metadata.labels or {} + kpi_labels = metadata.kpi_labels or {} + timing = metadata.timing or {} + + # Format path with link to metadata file + if get_file_link: + try: + metadata_file_link = get_file_link(metadata_file) + path_info = f"Path: [`{parent_dir}`]({metadata_file_link})" + except Exception as e: + logger.warning(f"Failed to create link for metadata file {metadata_file}: {e}") + path_info = f"Path: `{parent_dir}`" + else: + path_info = f"Path: `{parent_dir}`" + metadata_lines.append(f"* {path_info}") + + # Format labels + if labels: + label_items = [f"`{k}={v}`" for k, v in labels.items()] + metadata_lines.append(f"* Labels: {', '.join(label_items)}") + + # Format KPI labels + if kpi_labels: + kpi_label_items = [f"`{k}={v}`" for k, v in kpi_labels.items()] + metadata_lines.append(f"* KPI Labels: {', '.join(kpi_label_items)}") + + # Look for completion information in timing + completion_success = timing.get("completion", {}).get("success") + completion_message = timing.get("completion", {}).get("message", "") + + if completion_success is not None: + status_emoji = "✅" if completion_success else "❌" + message_part = f": `{completion_message}`" if completion_message else "" + metadata_lines.append(f"* Completion: {status_emoji}{message_part}") + + # Format test and benchmark duration + test_duration = timing.get("test_duration") + benchmark_duration = timing.get("benchmark_duration") + + if test_duration or benchmark_duration: + duration_parts = [] + if test_duration: + duration_parts.append(f"test: `{test_duration}`") + if benchmark_duration: + duration_parts.append(f"benchmark: `{benchmark_duration}`") + metadata_lines.append(f"* Duration: {', '.join(duration_parts)}") + + except Exception as e: + logger.warning(f"Failed to process caliper metadata file {metadata_file}: {e}") + metadata_lines.append(f"* `{metadata_file.parent}`: Error reading metadata - {e}") + + return "\n".join(metadata_lines) + + +def _format_caliper_metadata_info_for_step( + metadata_files: list[Path], get_file_link: Any, base_dir: Path +) -> list[str]: + """Format caliper metadata information for integration within step details.""" + if not metadata_files: + return [] + + metadata_lines = [] + + for metadata_file in metadata_files: + try: + # Load and parse metadata + with open(metadata_file, encoding="utf-8") as f: + metadata_dict = yaml.safe_load(f) + + metadata = CaliperTestMetadata.from_dict(metadata_dict) + + # Get relative path from base directory + try: + relative_path = metadata_file.parent.relative_to(base_dir) + display_path = str(relative_path) if str(relative_path) != "." else "root" + except ValueError: + # If relative_to fails, use the full path + display_path = str(metadata_file.parent) + + # Extract information + labels = metadata.labels or {} + kpi_labels = metadata.kpi_labels or {} + timing = metadata.timing or {} + + # Format path with link to metadata file + if get_file_link: + metadata_file_link = get_file_link(metadata_file) + path_info = f"* 📊 Test directory: [`{display_path}`]({metadata_file_link})" + else: + path_info = f"* 📊 Test directory: `{display_path}`" + metadata_lines.append(path_info) + + # Format labels + if labels: + label_items = [f"`{k}={v}`" for k, v in labels.items()] + metadata_lines.append(f" * {', '.join(label_items)}") + + # Format KPI labels + if kpi_labels: + kpi_label_items = [f"`{k}={v}`" for k, v in kpi_labels.items()] + metadata_lines.append(f" * KPI extra labels: {', '.join(kpi_label_items)}") + + # Look for completion information in timing + completion_success = timing.get("completion", {}).get("success") + completion_message = timing.get("completion", {}).get("message", "") + + if completion_success is not None: + status_emoji = "✅" if completion_success else "❌" + message_part = f": `{completion_message}`" if completion_message else "" + metadata_lines.append(f" * Completion: {status_emoji}{message_part}") + + # Format test and benchmark duration + test_duration = timing.get("test_duration") + benchmark_duration = timing.get("benchmark_duration") + + if test_duration or benchmark_duration: + duration_parts = [] + if test_duration: + duration_parts.append(f"test: `{test_duration}`") + if benchmark_duration: + duration_parts.append(f"benchmark: `{benchmark_duration}`") + metadata_lines.append(f" * Duration: {', '.join(duration_parts)}") + + except Exception as e: + logger.warning(f"Failed to process caliper metadata file {metadata_file}: {e}") + relative_path = ( + metadata_file.parent.relative_to(base_dir) if base_dir else metadata_file.parent + ) + metadata_lines.append( + f"* 📊 Test directory: `{relative_path}` - Error reading metadata: {e}" + ) + + return metadata_lines From ba23377c9ddabe635b1e80376044a96097dc931a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 20:28:17 +0200 Subject: [PATCH 0823/1072] [core] library: export: update the saved fjob --- projects/core/library/export.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 3e8cf0332..1e875b492 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -88,6 +88,20 @@ def _update_fjob_export_status(status: ExportStatus): patch_result = run.run(patch_cmd, check=False) if patch_result.returncode == 0: logger.info(f"Updated fjob/{fjob_name} status with export artifacts status") + + # Save a copy of the updated fjob to metadata directory + try: + metadata_dir = ci_lib.get_ci_metadata_dir() + fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" + fournos_fjob_path.parent.mkdir(parents=True, exist_ok=True) + + with open(fournos_fjob_path, "w", encoding="utf-8") as f: + yaml.dump(fjob_data, f, indent=2, default_flow_style=False) + + logger.info(f"Saved updated fjob copy to {fournos_fjob_path}") + + except Exception as e: + logger.warning(f"Failed to save fjob copy to metadata directory: {e}") else: logger.warning(f"Failed to update fjob status: {patch_cmd}") From 7b7ae9062d3529a638ef9172169ebae4678e8fe9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 20:31:49 +0200 Subject: [PATCH 0824/1072] [core] library: export: upload the final fjob after the file upload --- projects/core/library/export.py | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 1e875b492..80eaa7223 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -494,26 +494,26 @@ def _update_artifacts( # Collect files to upload files_to_upload = [] - # Check for run.log - log_file = artifact_dir_path / "run.log" - if log_file.is_file(): - files_to_upload.append(log_file) - - # Check for notification file - notif_file = artifact_dir_path / "NOTIFICATION-github.md" - if notif_file.is_file(): - files_to_upload.append(notif_file) + for fpath in [ + artifact_dir_path / "run.log", + artifact_dir_path / "NOTIFICATION-github.md", + artifact_dir_path / "000__ci_metadata" / "fournos_fjob.yaml", + ]: + if fpath.is_file(): + files_to_upload.append(fpath) # Upload files if any exist - if files_to_upload: - from projects.caliper.engine.file_export.mlflow_backend import update_artifacts + if not files_to_upload: + return - update_artifacts( - run_id=run_id, - files=dict.fromkeys(files_to_upload, artifact_path), - tracking_uri=tracking_uri, - connection=connection, - ) + from projects.caliper.engine.file_export.mlflow_backend import update_artifacts + + update_artifacts( + run_id=run_id, + files=dict.fromkeys(files_to_upload, artifact_path), + tracking_uri=tracking_uri, + connection=connection, + ) def caliper_export_list_vaults() -> list[str]: From db75dba01ae034f8a3d5acb540010f59d410c8d8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 20:32:21 +0200 Subject: [PATCH 0825/1072] [core] library: export_notifications: include the censoring details --- projects/core/library/export_notifications.py | 92 ++++++++++++++++--- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 9b844197f..e3cbf9391 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -1139,19 +1139,17 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> except Exception as e: logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") + # Create file link function for this step (shared by multiple extractors) + def get_file_link(file_path: Path) -> str: + if mlflow_run_url: + # Create MLflow artifact URL + return _create_mlflow_file_url_for_step(mlflow_run_url, step_dir.name, str(file_path)) + else: + # Fallback: just return the file path as text + return str(file_path) + # Extract caliper metadata for this specific step (before postprocess status) try: - # Create file link function for this step - def get_file_link(file_path: Path) -> str: - if mlflow_run_url: - # Create MLflow artifact URL - return _create_mlflow_file_url_for_step( - mlflow_run_url, step_dir.name, str(file_path) - ) - else: - # Fallback: just return the file path as text - return str(file_path) - metadata_files = _search_caliper_metadata_files(step_dir) metadata_info = _format_caliper_metadata_info_for_step( metadata_files, get_file_link, step_dir.parent if step_dir.parent else step_dir @@ -1161,6 +1159,14 @@ def get_file_link(file_path: Path) -> str: except Exception as e: logger.warning(f"Failed to extract caliper metadata for step {step_dir.name}: {e}") + # Extract censoring report for this specific step (before postprocess status) + try: + censoring_info = _format_censoring_report_info_for_step(step_dir, get_file_link) + if censoring_info: + step_details.extend(censoring_info) + except Exception as e: + logger.warning(f"Failed to extract censoring report for step {step_dir.name}: {e}") + # Extract postprocess status for this specific step try: postprocess_info = _extract_postprocess_status_info(step_dir) @@ -1521,6 +1527,70 @@ def _format_caliper_metadata_info(metadata_files: list[Path], get_file_link: Any return "\n".join(metadata_lines) +def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) -> list[str]: + """Format censoring report information for integration within step details.""" + censoring_report_path = step_dir / "censoring_report.yaml" + + if not censoring_report_path.exists(): + return [] + + try: + with open(censoring_report_path, encoding="utf-8") as f: + report_data = yaml.safe_load(f) + + if not report_data: + return [] + + censoring_lines = [] + + # Get censoring statistics + total_files = report_data.get("total_files", 0) + clean_files = report_data.get("clean_files", 0) + safe_censored_files = report_data.get("safe_censored_files", 0) + censored_files = report_data.get("censored_files", 0) # Unexpected censoring + + censoring_lines.append("* 🔒 Censoring Report") + # Create link to censoring report file + if get_file_link: + try: + report_link = get_file_link(censoring_report_path) + first_line = "[📊 {total_files} files scanned]({report_link})" + except Exception as e: + logger.warning( + f"Failed to create link for censoring report {censoring_report_path}: {e}" + ) + first_line = f"📊 {total_files} files scanned" + else: + first_line = f"📊 {total_files} files scanned" + + censoring_lines.append(f" * {first_line}") + + # Show breakdown of file types + if clean_files > 0: + censoring_lines.append(f" * ✅ Clean files: {clean_files}") + + if safe_censored_files > 0: + censoring_lines.append(f" * 🔐 Safe replacements: {safe_censored_files}") + + if censored_files > 0: + censoring_lines.append(f" * ⚠️ Unexpected censoring: {censored_files}") + + # Show details for unexpected censoring if available + censored_by_reason = report_data.get("censored_by_reason", {}) + if censored_by_reason: + for reason, files in censored_by_reason.items(): + file_count = len(files) + # Truncate reason if too long + display_reason = reason[:50] + "..." if len(reason) > 50 else reason + censoring_lines.append(f" * {display_reason}: {file_count} file(s)") + + return censoring_lines + + except Exception as e: + logger.warning(f"Failed to process censoring report {censoring_report_path}: {e}") + return [f"* 🔒 Censoring Report: Error reading report - `{e}`"] + + def _format_caliper_metadata_info_for_step( metadata_files: list[Path], get_file_link: Any, base_dir: Path ) -> list[str]: From f1a826bbd271c362ec500ffec018b7460927fff7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 20:49:49 +0200 Subject: [PATCH 0826/1072] [core] library: export_notifications: include the child runs --- projects/core/library/export_notifications.py | 76 +++++++------------ 1 file changed, 27 insertions(+), 49 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index e3cbf9391..eb885bfdf 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -39,6 +39,7 @@ class BackendResult: tracking_uri: str | None = None detail: str | None = None status: str | None = None + child_runs: list[dict[str, str]] | None = None @classmethod def from_dict(cls, data: dict[str, Any]) -> "BackendResult": @@ -57,6 +58,7 @@ def from_dict(cls, data: dict[str, Any]) -> "BackendResult": tracking_uri=data.get("tracking_uri"), detail=data.get("detail"), status=data.get("status"), + child_runs=data.get("child_runs"), ) def to_dict(self) -> dict[str, Any]: @@ -74,6 +76,8 @@ def to_dict(self) -> dict[str, Any]: result["detail"] = self.detail if self.status is not None: result["status"] = self.status + if self.child_runs is not None: + result["child_runs"] = self.child_runs return result @@ -90,10 +94,7 @@ def from_dict(cls, data: dict[str, Any]) -> "CaliperArtifactsExport": backends_data = data.get("backends", {}) backends = {} for backend_name, backend_data in backends_data.items(): - if isinstance(backend_data, dict): - backends[backend_name] = BackendResult.from_dict(backend_data) - else: - backends[backend_name] = backend_data # Keep non-dict values as-is + backends[backend_name] = BackendResult.from_dict(backend_data) return cls( version=data.get("version", 1), @@ -106,10 +107,7 @@ def to_dict(self) -> dict[str, Any]: if self.backends: backends_dict = {} for backend_name, backend_result in self.backends.items(): - if isinstance(backend_result, BackendResult): - backends_dict[backend_name] = backend_result.to_dict() - else: - backends_dict[backend_name] = backend_result + backends_dict[backend_name] = backend_result.to_dict() result["backends"] = backends_dict return result @@ -207,12 +205,7 @@ def from_dict(cls, data: dict[str, Any]) -> "ExportStatus": # Infer success from backend results if no explicit status backend_successes = [] for backend_result in caliper_export.backends.values(): - if isinstance(backend_result, BackendResult): - backend_successes.append(backend_result.success) - elif isinstance(backend_result, dict): - # Handle raw dict backends (shouldn't happen after conversion) - backend_status = backend_result.get("status", "") - backend_successes.append(backend_status == "success") + backend_successes.append(backend_result.success) # Overall success if all backends succeeded success = all(backend_successes) if backend_successes else False @@ -429,7 +422,7 @@ def _get_project_and_args(project: str) -> tuple[str, str]: logger.warning(f"Failed to read fournos job for project/args: {e}") if fjob_args_str: - fjob_args_str = f"with `{fjob_args_str}`" + fjob_args_str = f" | `{fjob_args_str}`" return fjob_project, fjob_args_str @@ -543,13 +536,13 @@ def _build_enhanced_notification( if artifact_links: notification_parts += ["", "---"] - notification_parts.append("**Artifact Links**") - notification_parts.extend([f"* {link}" for link in artifact_links]) + notification_parts.append("**MLFlow links**") + notification_parts.extend(artifact_links) else: if notification_success: - notification_parts.append("**Artifact Links:** No direct links available") + notification_parts.append("**MLFlow links:** No direct links available") else: - notification_parts.append("**Artifact Links:** Error extracting links") + notification_parts.append("**MLFlow links:** Error extracting links") if step_status: notification_parts += ["", "---"] @@ -902,38 +895,23 @@ def _extract_artifact_links(status: ExportStatus) -> tuple[list[str], str | None return artifact_links, mlflow_run_url for backend_name, backend_result in caliper_export.backends.items(): - if isinstance(backend_result, BackendResult): - # Use typed access for BackendResult objects - if backend_result.experiment_url: - artifact_links.append( - f"[{backend_name} Experiment]({backend_result.experiment_url})" - ) + if not isinstance(backend_result, BackendResult): + continue + # Use typed access for BackendResult objects - if backend_result.run_url: - mlflow_run_url = backend_result.run_url - artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") + if backend_result.run_url: + mlflow_run_url = backend_result.run_url + artifact_links.append(f"* [{backend_name} results]({mlflow_run_url})") - elif isinstance(backend_result, dict): - # Legacy support for dict-based backend results - if backend_result.get("experiment_url"): - artifact_links.append( - f"[{backend_name} Experiment]({backend_result['experiment_url']})" - ) - - if backend_result.get("run_url"): - mlflow_run_url = backend_result["run_url"] - artifact_links.append(f"[{backend_name} Results]({mlflow_run_url})") - elif backend_result.get("artifact_url"): - artifact_links.append( - f"[{backend_name} Artifacts]({backend_result['artifact_url']})" - ) - elif backend_result.get("dashboard_url"): - artifact_links.append( - f"[{backend_name} Dashboard]({backend_result['dashboard_url']})" - ) + # Add child runs if they exist + if not backend_result.child_runs: + continue - # Note: artifact_url is not part of the main export status - # This appears to be legacy code - removing for now + for child_run in backend_result.child_runs: + run_name = child_run.get("run_name") + run_url = child_run.get("run_url") + if run_name and run_url: + artifact_links.append(f" * nested run [{run_name}]({run_url})") return artifact_links, mlflow_run_url @@ -1554,7 +1532,7 @@ def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) - if get_file_link: try: report_link = get_file_link(censoring_report_path) - first_line = "[📊 {total_files} files scanned]({report_link})" + first_line = f"[📊 {total_files} files scanned]({report_link})" except Exception as e: logger.warning( f"Failed to create link for censoring report {censoring_report_path}: {e}" From 880866625aa98b32849f8c75db8dee3a493552e6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 09:34:19 +0200 Subject: [PATCH 0827/1072] vaults: psap-forge-mlflow-export: disable the MLFLOW URI censoring --- vaults/psap-forge-mlflow-export.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/vaults/psap-forge-mlflow-export.yaml b/vaults/psap-forge-mlflow-export.yaml index 6fb3e2208..56d12ee47 100644 --- a/vaults/psap-forge-mlflow-export.yaml +++ b/vaults/psap-forge-mlflow-export.yaml @@ -13,6 +13,7 @@ content: insecure_tls: true tracking_uri: + sensible: false # disable the URI censoring for the time being. We'll re-enable it when we switch the instance. censor_text: "https://mlflow_tracking_uri_hidden" description: | the URI of the MLFlow endpoint, so that the CI engine can censor it in the logs From 33337d8dd2ecc31e94531c7df21a81922f22b033 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 09:50:01 +0200 Subject: [PATCH 0828/1072] [core] library: export_notifications: always try to make the link relative to the step directory --- projects/core/library/export_notifications.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index eb885bfdf..fdd891b0d 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -1120,8 +1120,17 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> # Create file link function for this step (shared by multiple extractors) def get_file_link(file_path: Path) -> str: if mlflow_run_url: - # Create MLflow artifact URL - return _create_mlflow_file_url_for_step(mlflow_run_url, step_dir.name, str(file_path)) + # Create MLflow artifact URL - convert absolute path to relative to step directory + try: + relative_file_path = file_path.relative_to(step_dir) + return _create_mlflow_file_url_for_step( + mlflow_run_url, step_dir.name, str(relative_file_path) + ) + except ValueError: + # If file_path is not under step_dir, use the file name only + return _create_mlflow_file_url_for_step( + mlflow_run_url, step_dir.name, file_path.name + ) else: # Fallback: just return the file path as text return str(file_path) From 2e274a1a5e069ed31f78d66c6ecf7e94ac75c18e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 10:04:03 +0200 Subject: [PATCH 0829/1072] [core] library: export_notifications: fix the test directory details --- projects/core/library/export_notifications.py | 146 ++++++------------ 1 file changed, 51 insertions(+), 95 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index fdd891b0d..4b64fd7a9 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -28,6 +28,37 @@ logger = logging.getLogger(__name__) +def _calculate_duration(start_time: str, end_time: str) -> str: + """Calculate duration between two ISO timestamps and return formatted string.""" + try: + from datetime import datetime + + # Parse ISO timestamps (handle both with and without 'Z' suffix) + start_time_clean = ( + start_time.replace("Z", "+00:00") if start_time.endswith("Z") else start_time + ) + end_time_clean = end_time.replace("Z", "+00:00") if end_time.endswith("Z") else end_time + + start_dt = datetime.fromisoformat(start_time_clean) + end_dt = datetime.fromisoformat(end_time_clean) + + duration = end_dt - start_dt + total_seconds = duration.total_seconds() + + if total_seconds < 60: + return f"{total_seconds:.1f}s" + elif total_seconds < 3600: + minutes = total_seconds / 60 + return f"{minutes:.1f}m" + else: + hours = total_seconds / 3600 + return f"{hours:.1f}h" + + except Exception as e: + logger.warning(f"Failed to calculate duration between {start_time} and {end_time}: {e}") + return f"{start_time} → {end_time}" + + @dataclass class BackendResult: """Result from a single backend export.""" @@ -1440,80 +1471,6 @@ def _search_caliper_metadata_files(step_dir: Path) -> list[Path]: return metadata_files -def _format_caliper_metadata_info(metadata_files: list[Path], get_file_link: Any) -> str: - """Format caliper metadata information for display.""" - if not metadata_files: - return "" - - metadata_lines = [] - metadata_lines.append("**Test Directories**") - - for metadata_file in metadata_files: - try: - # Load and parse metadata - with open(metadata_file, encoding="utf-8") as f: - metadata_dict = yaml.safe_load(f) - - metadata = CaliperTestMetadata.from_dict(metadata_dict) - - # Get parent directory path - parent_dir = metadata_file.parent - - # Extract information - labels = metadata.labels or {} - kpi_labels = metadata.kpi_labels or {} - timing = metadata.timing or {} - - # Format path with link to metadata file - if get_file_link: - try: - metadata_file_link = get_file_link(metadata_file) - path_info = f"Path: [`{parent_dir}`]({metadata_file_link})" - except Exception as e: - logger.warning(f"Failed to create link for metadata file {metadata_file}: {e}") - path_info = f"Path: `{parent_dir}`" - else: - path_info = f"Path: `{parent_dir}`" - metadata_lines.append(f"* {path_info}") - - # Format labels - if labels: - label_items = [f"`{k}={v}`" for k, v in labels.items()] - metadata_lines.append(f"* Labels: {', '.join(label_items)}") - - # Format KPI labels - if kpi_labels: - kpi_label_items = [f"`{k}={v}`" for k, v in kpi_labels.items()] - metadata_lines.append(f"* KPI Labels: {', '.join(kpi_label_items)}") - - # Look for completion information in timing - completion_success = timing.get("completion", {}).get("success") - completion_message = timing.get("completion", {}).get("message", "") - - if completion_success is not None: - status_emoji = "✅" if completion_success else "❌" - message_part = f": `{completion_message}`" if completion_message else "" - metadata_lines.append(f"* Completion: {status_emoji}{message_part}") - - # Format test and benchmark duration - test_duration = timing.get("test_duration") - benchmark_duration = timing.get("benchmark_duration") - - if test_duration or benchmark_duration: - duration_parts = [] - if test_duration: - duration_parts.append(f"test: `{test_duration}`") - if benchmark_duration: - duration_parts.append(f"benchmark: `{benchmark_duration}`") - metadata_lines.append(f"* Duration: {', '.join(duration_parts)}") - - except Exception as e: - logger.warning(f"Failed to process caliper metadata file {metadata_file}: {e}") - metadata_lines.append(f"* `{metadata_file.parent}`: Error reading metadata - {e}") - - return "\n".join(metadata_lines) - - def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) -> list[str]: """Format censoring report information for integration within step details.""" censoring_report_path = step_dir / "censoring_report.yaml" @@ -1606,7 +1563,7 @@ def _format_caliper_metadata_info_for_step( # Extract information labels = metadata.labels or {} kpi_labels = metadata.kpi_labels or {} - timing = metadata.timing or {} + timing = metadata.timing # Format path with link to metadata file if get_file_link: @@ -1616,6 +1573,14 @@ def _format_caliper_metadata_info_for_step( path_info = f"* 📊 Test directory: `{display_path}`" metadata_lines.append(path_info) + # Look for completion information + if metadata.completion: + status_emoji = "✅" if metadata.completion.success else "❌" + message_part = ( + f" `{metadata.completion.message}`" if metadata.completion.message else "" + ) + metadata_lines.append(f" * {status_emoji}{message_part}") + # Format labels if labels: label_items = [f"`{k}={v}`" for k, v in labels.items()] @@ -1626,26 +1591,17 @@ def _format_caliper_metadata_info_for_step( kpi_label_items = [f"`{k}={v}`" for k, v in kpi_labels.items()] metadata_lines.append(f" * KPI extra labels: {', '.join(kpi_label_items)}") - # Look for completion information in timing - completion_success = timing.get("completion", {}).get("success") - completion_message = timing.get("completion", {}).get("message", "") - - if completion_success is not None: - status_emoji = "✅" if completion_success else "❌" - message_part = f": `{completion_message}`" if completion_message else "" - metadata_lines.append(f" * Completion: {status_emoji}{message_part}") - - # Format test and benchmark duration - test_duration = timing.get("test_duration") - benchmark_duration = timing.get("benchmark_duration") - - if test_duration or benchmark_duration: - duration_parts = [] - if test_duration: - duration_parts.append(f"test: `{test_duration}`") - if benchmark_duration: - duration_parts.append(f"benchmark: `{benchmark_duration}`") - metadata_lines.append(f" * Duration: {', '.join(duration_parts)}") + # Format timing information - show durations instead of timestamps + duration_parts = [] + + if timing: + for phase_name, timing_entry in timing.phases.items(): + if timing_entry.end: # Only show duration if we have both start and end + duration = _calculate_duration(timing_entry.start, timing_entry.end) + duration_parts.append(f"`{phase_name}: {duration}`") + + if duration_parts: + metadata_lines.append(f" * Duration {', '.join(duration_parts)}") except Exception as e: logger.warning(f"Failed to process caliper metadata file {metadata_file}: {e}") From 2e583ea5b66bfbd529afe9e3a40cc1d7a470a550 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 10:13:40 +0200 Subject: [PATCH 0830/1072] [core] library: export_notifications: include the test metadata listing --- .../skeleton/orchestration/test_skeleton.py | 238 ++++++++++++++---- 1 file changed, 187 insertions(+), 51 deletions(-) diff --git a/projects/skeleton/orchestration/test_skeleton.py b/projects/skeleton/orchestration/test_skeleton.py index a29a4e42b..441930c12 100644 --- a/projects/skeleton/orchestration/test_skeleton.py +++ b/projects/skeleton/orchestration/test_skeleton.py @@ -3,37 +3,102 @@ import pathlib import signal import time +from datetime import UTC, datetime import yaml -from projects.core.library import config, env, vault +from projects.caliper.engine.constants import METADATA_FILE +from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.skeleton.toolbox.cluster_info.main import run as cluster_info logger = logging.getLogger(__name__) -def seed_skeleton_caliper_artifacts() -> pathlib.Path: - """ - Create minimal Caliper inputs under the FORGE artifact root: +def get_iso_timestamp() -> str: + """Get current timestamp in ISO format with Z timezone.""" + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def create_seed_data_with_completion( + test_start_time: str, + test_end_time: str, + benchmark_start_time: str, + benchmark_end_time: str, + cluster_info_start_time: str | None, + cluster_info_end_time: str | None, + success: bool, + message: str, +) -> None: + """Create seed data with complete timing and completion information.""" + + # Build timing structure + timing_data = { + "test": {"start": test_start_time, "end": test_end_time}, + "benchmark": {"start": benchmark_start_time, "end": benchmark_end_time}, + } + + # Add cluster info timing if available + if cluster_info_start_time and cluster_info_end_time: + timing_data["cluster_info"] = { + "start": cluster_info_start_time, + "end": cluster_info_end_time, + } + + # Build completion data + completion_data = {"success": success, "message": message} + + # Create the seed data with timing and completion + with env.NextArtifactDir("skeleton_seed_data_for_caliper_postprocessing"): + seed_skeleton_caliper_artifacts_with_data(timing_data, completion_data) - * ``__caliper_test_metadata__.yaml`` + ``metrics.json`` per scenario (required by the skeleton plugin). - """ + +def seed_skeleton_caliper_artifacts_with_data( + timing_data: dict, completion_data: dict +) -> pathlib.Path: + """Create minimal Caliper inputs with timing and completion data.""" demo_dir = env.ARTIFACT_DIR + + skeleton_config = config.project.get_config("skeleton", print=False) + + # Base labels for all scenarios + base_labels = { + "test": "skeleton", + "deep_testing": str(skeleton_config.get("deep_testing", False)), + "collect_cluster_info": str(skeleton_config.get("collect_cluster_info", True)), + } + FAKE_DATA = ( ("smoke", 120.5, 8.2), ("load", 87.0, 22.1), ) + for scenario, throughput, latency_ms in FAKE_DATA: d = demo_dir / scenario d.mkdir(parents=True, exist_ok=True) - write_test_labels(d, {"scenario": scenario}, dump_config=False) + + # Combine base labels with scenario-specific label + scenario_labels = {**base_labels, "scenario": scenario} + + # Create metadata with timing and completion data + write_test_labels(d, scenario_labels, dump_config=False, timing=timing_data) + + # Add completion data to the metadata file + scenario_metadata_path = d / METADATA_FILE + if scenario_metadata_path.exists(): + with scenario_metadata_path.open("r", encoding="utf-8") as f: + scenario_data = yaml.safe_load(f) + scenario_data["completion"] = completion_data + with scenario_metadata_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(scenario_data, f, sort_keys=False) + logger.info(f"Created {scenario} metadata with timing and completion data") + (d / "metrics.json").write_text( json.dumps({"throughput": throughput, "latency_ms": latency_ms}), encoding="utf-8", ) - logger.info("Seeded Caliper demo tree under %s", demo_dir) + logger.info("Seeded Caliper demo tree under %s with complete timing data", demo_dir) return demo_dir @@ -96,53 +161,124 @@ def skeleton_take_time(): def do_test(): logger.info("=== Skeleton Project Test Phase ===") - if config.project.get_config("skeleton.deep_testing"): - logger.info("Running the (fake) deep testing ...") - else: - logger.info("Running the (fake) light testing ...") - - client_id = vault.get_vault_content_path("psap-forge-notifications", "topsail-bot.clientid") - if not client_id: - logger.error("`client_id` secret not available.") - else: - logger.info(f"`client_id` secret available. Size: {client_id.stat().st_size}b") - del client_id + # Capture test timing + test_start_time = get_iso_timestamp() - skeleton_config = config.project.get_config("skeleton", print=False) - - yaml_cfg = yaml.dump( - {"skeleton": skeleton_config}, - indent=4, - default_flow_style=False, - sort_keys=False, - ) - logger.info("") - logger.info(f"Fake test configuration:\n{yaml_cfg}") - - skeleton_take_time() - - with env.NextArtifactDir("skeleton_seed_data_for_caliper_postprocessing"): - seed_skeleton_caliper_artifacts() + try: + if config.project.get_config("skeleton.deep_testing"): + logger.info("Running the (fake) deep testing ...") + else: + logger.info("Running the (fake) light testing ...") + + skeleton_config = config.project.get_config("skeleton", print=False) + + yaml_cfg = yaml.dump( + {"skeleton": skeleton_config}, + indent=4, + default_flow_style=False, + sort_keys=False, + ) + logger.info("") + logger.info(f"Fake test configuration:\n{yaml_cfg}") + + # Capture benchmark timing + benchmark_start_time = get_iso_timestamp() + try: + skeleton_take_time() + finally: + benchmark_end_time = get_iso_timestamp() + + cluster_info_start_time = None + cluster_info_end_time = None + + if config.project.get_config("skeleton.collect_cluster_info"): + # Add timing around cluster info gathering + cluster_info_start_time = get_iso_timestamp() + try: + # Demonstrate calling a toolbox from orchestration + logger.info("Running cluster information toolbox...") + + result = cluster_info(output_format="text") + + if not result: + logger.warning("⚠️ Cluster information gathering didn't work") + test_end_time = get_iso_timestamp() + create_seed_data_with_completion( + test_start_time, + test_end_time, + benchmark_start_time, + benchmark_end_time, + cluster_info_start_time, + get_iso_timestamp(), + False, + "Cluster information gathering failed", + ) + return 1 + + cluster_nodes_dest = getattr(result, "cluster_nodes_dest", None) + if not cluster_nodes_dest: + logger.warning( + "⚠️ Cluster information gathering didn't generate the cluster node file" + ) + test_end_time = get_iso_timestamp() + create_seed_data_with_completion( + test_start_time, + test_end_time, + benchmark_start_time, + benchmark_end_time, + cluster_info_start_time, + get_iso_timestamp(), + False, + "Cluster information gathering incomplete - no node file generated", + ) + return 1 + + finally: + cluster_info_end_time = get_iso_timestamp() + + logger.info("✅ Cluster information gathering completed successfully") + + test_end_time = get_iso_timestamp() + + # Determine completion message based on cluster info setting + if not config.project.get_config("skeleton.collect_cluster_info"): + completion_message = "Test completed successfully (cluster info disabled)" + else: + completion_message = "Test completed successfully" + + # Create seed data with all timing and completion information + create_seed_data_with_completion( + test_start_time, + test_end_time, + benchmark_start_time, + benchmark_end_time, + cluster_info_start_time, + cluster_info_end_time, + True, + completion_message, + ) - if not config.project.get_config("skeleton.collect_cluster_info"): - logger.info("⚠️ Cluster information gathering not enabled. Returning early.") return 0 - # Demonstrate calling a toolbox from orchestration - logger.info("Running cluster information toolbox...") - - result = cluster_info(output_format="text") - - if not result: - logger.warning("⚠️ Cluster information gathering didn't work") - return 1 - - cluster_nodes_dest = getattr(result, "cluster_nodes_dest", None) - if not cluster_nodes_dest: - logger.warning("⚠️ Cluster information gathering didn't generate the cluster node file") - return 1 - - logger.info("✅ Cluster information gathering completed successfully") + except Exception as e: + logger.exception("❌ Test failed with exception") + test_end_time = get_iso_timestamp() + + # Create seed data even after failure so timing data is preserved + try: + create_seed_data_with_completion( + test_start_time, + test_end_time, + locals().get("benchmark_start_time"), + locals().get("benchmark_end_time"), + locals().get("cluster_info_start_time"), + locals().get("cluster_info_end_time"), + False, + f"Test failed: {e}", + ) + except Exception as seed_error: + logger.warning(f"Failed to create seed data after test failure: {seed_error}") + raise logger.info(f"Check {cluster_nodes_dest.parent} directory for detailed cluster information.") return 0 From 1505785d5db7569720d70dcbab9d60fccc93f18e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 10:49:30 +0200 Subject: [PATCH 0831/1072] [skeleton] orchestration: config: disable the ai-data eval generation for the time being --- projects/skeleton/orchestration/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 48e3f46d1..1fd65120c 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -73,7 +73,7 @@ caliper: enabled: false output: kpis/dashboard.csv artifacts_to_ai_data: - enabled: true + enabled: false output_dir: ai_data s3: From ac99d1556015ef07f07daa9d66496485e05778fc Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 11:51:02 +0200 Subject: [PATCH 0832/1072] [caliper] engine: more dataclasses --- docs/caliper/test-metadata-format.md | 22 ++++ projects/caliper/engine/kpi/dataclasses.py | 130 +++++++++++++++++++-- 2 files changed, 144 insertions(+), 8 deletions(-) diff --git a/docs/caliper/test-metadata-format.md b/docs/caliper/test-metadata-format.md index 843382f10..805f69d4a 100644 --- a/docs/caliper/test-metadata-format.md +++ b/docs/caliper/test-metadata-format.md @@ -18,6 +18,17 @@ kpi_labels: platform: "CKS" gpu_type: "H100" test_harness: "guidellm" +timing: + # Test timing information (optional) + test: + start: "2024-03-15T10:30:00.123456Z" # ISO timestamp + end: "2024-03-15T10:35:30.789012Z" # optional during execution + benchmark: + start: "2024-03-15T10:32:00.456789Z" + end: "2024-03-15T10:35:00.123456Z" + cluster_info: # additional phases as needed + start: "2024-03-15T10:35:30.789012Z" + end: "2024-03-15T10:36:00.123456Z" completion: # Test execution status (added at completion) success: true|false @@ -38,6 +49,10 @@ completion: - **`run_id`**: MLflow run ID (assigned by the server during pre-creation) - **`experiment_id`** *(optional)*: MLflow experiment ID - **`workspace`** *(optional)*: MLflow workspace name +- **`timing`** *(optional)*: Test execution timing data with named phases + - Each phase contains **`start`** (required) and **`end`** (optional during execution) ISO timestamps + - Common phases: **`test`**, **`benchmark`**, **`cluster_info`** + - Phases are flexible - any meaningful timing section can be added - **`completion`**: Test execution status - **`success`**: `true` if succeeded, `false` if failed - **`message`**: Human-readable status description @@ -99,6 +114,13 @@ mlflow_destination: run_id: "48e49dfc966c487cb76cf105a5314908" experiment_id: "264" workspace: "forge-rhaiis" +timing: + test: + start: "2024-03-15T10:30:00.123456Z" + end: "2024-03-15T10:35:30.789012Z" + benchmark: + start: "2024-03-15T10:32:00.456789Z" + end: "2024-03-15T10:35:00.123456Z" completion: success: true message: "Test completed successfully" diff --git a/projects/caliper/engine/kpi/dataclasses.py b/projects/caliper/engine/kpi/dataclasses.py index 658e34930..ff0100180 100644 --- a/projects/caliper/engine/kpi/dataclasses.py +++ b/projects/caliper/engine/kpi/dataclasses.py @@ -301,15 +301,107 @@ def from_dict(cls, data: dict[str, Any]) -> HierarchicalKpiFormat: ) +@dataclass +class TimingEntry: + """Timing entry with start and optional end timestamps.""" + + start: str + end: str | None = None + + def to_dict(self) -> dict[str, str | None]: + """Convert to dictionary for YAML serialization.""" + result = {"start": self.start} + if self.end is not None: + result["end"] = self.end + return result + + @classmethod + def from_dict(cls, data: dict[str, str]) -> TimingEntry: + """Create TimingEntry from dictionary data.""" + return cls(start=data["start"], end=data.get("end")) + + +@dataclass +class TimingData: + """Test timing data with flexible named phases.""" + + phases: dict[str, TimingEntry] = field(default_factory=dict) + + def to_dict(self) -> dict[str, dict[str, str]]: + """Convert to dictionary for YAML serialization.""" + return {phase_name: timing.to_dict() for phase_name, timing in self.phases.items()} + + @classmethod + def from_dict(cls, data: dict[str, dict[str, str]]) -> TimingData: + """Create TimingData from dictionary data.""" + phases = {} + for phase_name, timing_dict in data.items(): + if isinstance(timing_dict, dict) and "start" in timing_dict: + phases[phase_name] = TimingEntry.from_dict(timing_dict) + return cls(phases=phases) + + def get_phase(self, name: str) -> TimingEntry | None: + """Get timing entry for a specific phase.""" + return self.phases.get(name) + + def set_phase(self, name: str, start: str, end: str | None = None) -> None: + """Set timing entry for a specific phase.""" + self.phases[name] = TimingEntry(start=start, end=end) + + +@dataclass +class CompletionData: + """Test completion status with success flag and message.""" + + success: bool + message: str + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary for YAML serialization.""" + return {"success": self.success, "message": self.message} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> CompletionData: + """Create CompletionData from dictionary data.""" + return cls(success=data["success"], message=data["message"]) + + +@dataclass +class MlflowDestination: + """MLflow destination information for test runs.""" + + run_id: str + experiment_id: str + workspace: str = "" + + def to_dict(self) -> dict[str, str]: + """Convert to dictionary for YAML serialization.""" + return { + "run_id": self.run_id, + "experiment_id": self.experiment_id, + "workspace": self.workspace, + } + + @classmethod + def from_dict(cls, data: dict[str, str]) -> MlflowDestination: + """Create MlflowDestination from dictionary data.""" + return cls( + run_id=data["run_id"], + experiment_id=data["experiment_id"], + workspace=data.get("workspace", ""), + ) + + @dataclass class CaliperTestMetadata: """Caliper test metadata structure for __caliper_test_metadata__.yaml files.""" - version: str labels: dict[str, str] + version: str = "1" kpi_labels: dict[str, str] | None = None - mlflow_destination: dict[str, str] | None = None - timing: dict[str, Any] | None = None + mlflow_destination: MlflowDestination | None = None + timing: TimingData | None = None + completion: CompletionData | None = None def to_dict(self) -> dict[str, Any]: """Convert to dictionary for YAML serialization.""" @@ -320,20 +412,38 @@ def to_dict(self) -> dict[str, Any]: if self.kpi_labels is not None: result["kpi_labels"] = self.kpi_labels if self.mlflow_destination is not None: - result["mlflow_destination"] = self.mlflow_destination + result["mlflow_destination"] = self.mlflow_destination.to_dict() if self.timing is not None: - result["timing"] = self.timing + result["timing"] = self.timing.to_dict() + if self.completion is not None: + result["completion"] = self.completion.to_dict() return result @classmethod def from_dict(cls, data: dict[str, Any]) -> CaliperTestMetadata: """Create CaliperTestMetadata from dictionary data.""" + # Parse mlflow_destination if present + mlflow_dest = None + if data.get("mlflow_destination"): + mlflow_dest = MlflowDestination.from_dict(data["mlflow_destination"]) + + # Parse timing if present + timing = None + if data.get("timing"): + timing = TimingData.from_dict(data["timing"]) + + # Parse completion if present + completion = None + if data.get("completion"): + completion = CompletionData.from_dict(data["completion"]) + return cls( - version=data["version"], labels=data["labels"], + version=data.get("version", "1"), kpi_labels=data.get("kpi_labels"), - mlflow_destination=data.get("mlflow_destination"), - timing=data.get("timing"), + mlflow_destination=mlflow_dest, + timing=timing, + completion=completion, ) @@ -347,4 +457,8 @@ def from_dict(cls, data: dict[str, Any]) -> CaliperTestMetadata: "HierarchicalTestEntry", "HierarchicalKpiFormat", "CaliperTestMetadata", + "TimingEntry", + "TimingData", + "CompletionData", + "MlflowDestination", ] From 4bafef1ff380f7a555fe20a13dfffd8a5843200f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 11:52:28 +0200 Subject: [PATCH 0833/1072] [llm_d] orchestration: test_phase: use the timing and completion dataclasses --- projects/llm_d/orchestration/test_phase.py | 69 ++++++++++++++-------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index c82bd187b..68e9fb207 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -10,6 +10,12 @@ import yaml from projects.caliper.engine.constants import METADATA_FILE +from projects.caliper.engine.kpi.dataclasses import ( + CaliperTestMetadata, + CompletionData, + MlflowDestination, + TimingData, +) from projects.cluster.toolbox.capture_prometheus.main import run as capture_prometheus from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.dsl import shell @@ -190,14 +196,17 @@ def create_test_labels( kpi_labels = extract_kpi_labels_from_config() # Create initial timing structure with test start - timing = {"test": {"start": get_iso_timestamp()}} + timing_data = TimingData() + timing_data.set_phase("test", get_iso_timestamp()) # Start time only write_test_labels( env.ARTIFACT_DIR, labels, kpi_labels=kpi_labels if kpi_labels else None, - mlflow_destination=mlflow_destination, - timing=timing, + mlflow_destination=MlflowDestination.from_dict(mlflow_destination) + if mlflow_destination + else None, + timing=timing_data, ) logger.info("Created test labels with start time: %s", labels) @@ -239,23 +248,34 @@ def update_test_labels_with_timing(timing_section: str, timing_event: str) -> da logging.error("Caliper metadata file not found ...") return datetime.now(UTC) - # Read existing labels + # Read existing labels and parse with dataclass with test_labels_path.open("r", encoding="utf-8") as f: data = yaml.safe_load(f) - # Ensure timing section exists - if "timing" not in data: - data["timing"] = {} + metadata = CaliperTestMetadata.from_dict(data) - if timing_section not in data["timing"]: - data["timing"][timing_section] = {} + # Ensure timing exists + if not metadata.timing: + metadata.timing = TimingData() - # Add the timing event - data["timing"][timing_section][timing_event] = timestamp + # Get or create the phase + phase = metadata.timing.get_phase(timing_section) + if not phase: + if timing_event == "start": + metadata.timing.set_phase(timing_section, timestamp) + else: + # If we're setting end but no start exists, create with empty start + metadata.timing.set_phase(timing_section, "", timestamp) + else: + # Update existing phase + if timing_event == "start": + phase.start = timestamp + elif timing_event == "end": + phase.end = timestamp # Write updated labels with test_labels_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(data, f, sort_keys=False) + yaml.safe_dump(metadata.to_dict(), f, sort_keys=False) logger.info( "Updated test labels with timing: %s.%s = %s", timing_section, timing_event, timestamp @@ -281,24 +301,27 @@ def update_test_labels_with_status(success: bool, message: str) -> None: with test_labels_path.open("r", encoding="utf-8") as f: data = yaml.safe_load(f) - # Add completion information as separate top-level field - data["completion"] = { - "success": success, - "message": message, - } + metadata = CaliperTestMetadata.from_dict(data) + + # Add completion information using dataclass + metadata.completion = CompletionData(success=success, message=message) # Add test end timing test_end_time = get_iso_timestamp() - if "timing" not in data: - data["timing"] = {} - if "test" not in data["timing"]: - data["timing"]["test"] = {} + if not metadata.timing: + metadata.timing = TimingData() - data["timing"]["test"]["end"] = test_end_time + # Get or update the test phase end time + test_phase = metadata.timing.get_phase("test") + if test_phase: + test_phase.end = test_end_time + else: + # If test phase doesn't exist, create it with empty start and the end time + metadata.timing.set_phase("test", "", test_end_time) # Write updated labels with test_labels_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(data, f, sort_keys=False) + yaml.safe_dump(metadata.to_dict(), f, sort_keys=False) logger.info( "Updated test labels with completion status and end time: success=%s, message=%s", From 29a8519426be653a32224ea36fa2178edbbdc079 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 11:53:03 +0200 Subject: [PATCH 0834/1072] [skeleton] orchestration: test_skeleton: use the timing/completion dataclasses --- .../skeleton/orchestration/test_skeleton.py | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/projects/skeleton/orchestration/test_skeleton.py b/projects/skeleton/orchestration/test_skeleton.py index 441930c12..5fe3e1796 100644 --- a/projects/skeleton/orchestration/test_skeleton.py +++ b/projects/skeleton/orchestration/test_skeleton.py @@ -7,7 +7,7 @@ import yaml -from projects.caliper.engine.constants import METADATA_FILE +from projects.caliper.engine.kpi.dataclasses import CompletionData, TimingData from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.skeleton.toolbox.cluster_info.main import run as cluster_info @@ -32,21 +32,17 @@ def create_seed_data_with_completion( ) -> None: """Create seed data with complete timing and completion information.""" - # Build timing structure - timing_data = { - "test": {"start": test_start_time, "end": test_end_time}, - "benchmark": {"start": benchmark_start_time, "end": benchmark_end_time}, - } + # Build timing structure using dataclasses + timing_data = TimingData() + timing_data.set_phase("test", test_start_time, test_end_time) + timing_data.set_phase("benchmark", benchmark_start_time, benchmark_end_time) # Add cluster info timing if available if cluster_info_start_time and cluster_info_end_time: - timing_data["cluster_info"] = { - "start": cluster_info_start_time, - "end": cluster_info_end_time, - } + timing_data.set_phase("cluster_info", cluster_info_start_time, cluster_info_end_time) - # Build completion data - completion_data = {"success": success, "message": message} + # Build completion data using dataclasses + completion_data = CompletionData(success=success, message=message) # Create the seed data with timing and completion with env.NextArtifactDir("skeleton_seed_data_for_caliper_postprocessing"): @@ -54,7 +50,7 @@ def create_seed_data_with_completion( def seed_skeleton_caliper_artifacts_with_data( - timing_data: dict, completion_data: dict + timing_data: TimingData, completion_data: CompletionData ) -> pathlib.Path: """Create minimal Caliper inputs with timing and completion data.""" demo_dir = env.ARTIFACT_DIR @@ -74,26 +70,23 @@ def seed_skeleton_caliper_artifacts_with_data( ) for scenario, throughput, latency_ms in FAKE_DATA: - d = demo_dir / scenario - d.mkdir(parents=True, exist_ok=True) + dest_dir = demo_dir / scenario + dest_dir.mkdir(parents=True, exist_ok=True) # Combine base labels with scenario-specific label scenario_labels = {**base_labels, "scenario": scenario} - # Create metadata with timing and completion data - write_test_labels(d, scenario_labels, dump_config=False, timing=timing_data) - - # Add completion data to the metadata file - scenario_metadata_path = d / METADATA_FILE - if scenario_metadata_path.exists(): - with scenario_metadata_path.open("r", encoding="utf-8") as f: - scenario_data = yaml.safe_load(f) - scenario_data["completion"] = completion_data - with scenario_metadata_path.open("w", encoding="utf-8") as f: - yaml.safe_dump(scenario_data, f, sort_keys=False) - logger.info(f"Created {scenario} metadata with timing and completion data") + # Create metadata with timing and completion data using write_test_labels + write_test_labels( + dest_dir, + scenario_labels, + timing=timing_data, + completion=completion_data, + dump_config=False, + ) + logger.info(f"Created {scenario} metadata with timing and completion data") - (d / "metrics.json").write_text( + (dest_dir / "metrics.json").write_text( json.dumps({"throughput": throughput, "latency_ms": latency_ms}), encoding="utf-8", ) From 9ab7973c9f374755334a539f2d775ae9a6492932 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 13:34:23 +0200 Subject: [PATCH 0835/1072] [core] library: postprocess: use caliper dataclasses --- projects/core/library/postprocess.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index 60e61a27e..1e2475fb2 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -17,7 +17,12 @@ from pydantic import ValidationError from projects.caliper.engine.constants import METADATA_FILE -from projects.caliper.engine.kpi.dataclasses import CaliperTestMetadata +from projects.caliper.engine.kpi.dataclasses import ( + CaliperTestMetadata, + CompletionData, + MlflowDestination, + TimingData, +) from projects.caliper.orchestration.postprocess import ( run_postprocess_from_orchestration_config, ) @@ -40,8 +45,9 @@ def write_test_labels( version: str = "1", dump_config: bool = True, kpi_labels: dict[str, str] | None = None, - mlflow_destination: dict[str, str] | None = None, - timing: dict[str, Any] | None = None, + mlflow_destination: MlflowDestination | None = None, + timing: TimingData | None = None, + completion: CompletionData | None = None, ) -> Path: """Write Caliper test metadata files to mark a directory as a Caliper test base. @@ -54,8 +60,9 @@ def write_test_labels( version: Version string for the test labels format (default: "1") dump_config: Whether to save project configuration to config.yaml (default: True) kpi_labels: Optional dictionary of KPI labels for system context - mlflow_destination: Optional MLflow run destination (run_id, experiment_id, workspace) - timing: Optional dictionary of timing information for test phases + mlflow_destination: Optional MLflow run destination dataclass + timing: Optional timing data with test phases + completion: Optional test completion status Returns: Path to the created caliper metadata file file @@ -76,11 +83,12 @@ def write_test_labels( """ # Create typed metadata structure metadata = CaliperTestMetadata( - version=version, labels=labels, + version=version, kpi_labels=kpi_labels, mlflow_destination=mlflow_destination, timing=timing, + completion=completion, ) # Convert to dictionary for YAML serialization From 84618cee732d613d33faf33c39d827170cd9b531 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 13:36:35 +0200 Subject: [PATCH 0836/1072] [rhaiis] orchestration: test_phase: use the dataclasses --- projects/rhaiis/orchestration/test_phase.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 748f5608c..b1d50ee89 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -8,6 +8,7 @@ import yaml +from projects.caliper.engine.kpi.dataclasses import MlflowDestination from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.rhaiis.orchestration import runtime_config @@ -519,7 +520,13 @@ def _create_test_labels( "run_uuid": run_uuid, } - write_test_labels(env.ARTIFACT_DIR, labels, mlflow_destination=mlflow_destination) + write_test_labels( + env.ARTIFACT_DIR, + labels, + mlflow_destination=MlflowDestination.from_dict(mlflow_destination) + if mlflow_destination + else None, + ) logger.info("Created test labels: %s", labels) From 4384bdf0875af51989e696ff8fcc8eb6f407bf5c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 11 Sep 2026 14:20:59 +0200 Subject: [PATCH 0837/1072] [core] ci_entrypoint: prepare_ci: Run uv pip freeze using the run library with cache disabled --- projects/core/ci_entrypoint/prepare_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 893a78d03..01918677b 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -710,9 +710,9 @@ def save_pip_freeze(): logger.warning("uv not found, cannot run pip freeze") return - # Run uv pip freeze using the run library + # Run uv pip freeze using the run library with cache disabled result = run.run( - "uv pip freeze", + "UV_NO_CACHE=1 uv pip freeze", capture_stdout=True, check=False, ) From 6de6c3a0c57fe19e9533d96618d5e361977e213b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 09:15:26 +0200 Subject: [PATCH 0838/1072] [core] agentic: models: add a 5min timeout --- projects/core/agentic/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/core/agentic/models.py b/projects/core/agentic/models.py index 44ee3b601..994ca3d99 100644 --- a/projects/core/agentic/models.py +++ b/projects/core/agentic/models.py @@ -100,6 +100,8 @@ def create_llm_client(model_config: dict): http_client=http_client, # Disable streaming to match simpler request format streaming=False, + # 5-minute timeout for LLM requests + request_timeout=300, ) return llm From c50cb1f74635306e9ea9c6a39c0ceee5c67b344b Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 14 Sep 2026 09:23:21 -0400 Subject: [PATCH 0839/1072] [rhaiis] Add old-zeus cluster preset --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index acf09c811..183b05c74 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -9,3 +9,8 @@ zeus: rhaiis.cluster_tag: "zeus2" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 + +old-zeus: + rhaiis.cluster_tag: "old-zeus" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 From ca99bb9be10d644218a5a53bebfd3640fc1457ba Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Mon, 14 Sep 2026 09:36:08 -0400 Subject: [PATCH 0840/1072] [rhaiis] Correct old-zeus cluster tag --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 183b05c74..f43c75839 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -11,6 +11,6 @@ zeus: benchmarks.guidellm.fs_group: 0 old-zeus: - rhaiis.cluster_tag: "old-zeus" + rhaiis.cluster_tag: "zeus1" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 From 078a91cf96e08e50d1b76779aaff20852387f530 Mon Sep 17 00:00:00 2001 From: ashtarkb Date: Mon, 14 Sep 2026 18:13:48 +0300 Subject: [PATCH 0841/1072] add submit pages for rhaiis and llm_d --- projects/llm_d/ui/submit.yaml | 93 +++++++++++++ projects/rhaiis/ui/submit.yaml | 232 +++++++++++++++++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 projects/llm_d/ui/submit.yaml create mode 100644 projects/rhaiis/ui/submit.yaml diff --git a/projects/llm_d/ui/submit.yaml b/projects/llm_d/ui/submit.yaml new file mode 100644 index 000000000..374fa342c --- /dev/null +++ b/projects/llm_d/ui/submit.yaml @@ -0,0 +1,93 @@ +schema_version: 1 +project: llm_d +title: llm-d +description: > + llm-d inference-serving performance testing (vLLM-based distributed + serving: prefix-caching, disaggregated prefill/decode, intelligent + routing). Each dimension below is its own presets.d file — unlike + RHAIIS, llm_d's presets aren't self-tagged/mixed, so every field sources + its options straight from its own file (options_ref + preset_pool, see + docs/ui-schema-spec.md in psap-control-center) instead of a shared + presets_ref pool. + +modes: + - id: single + label: Submit Job + default: true + sections: + - id: core + label: Core + fields: + - key: preset + label: Preset + type: select + required: true + help: > + Deployment/smoke shape to run — see + orchestration/presets.d/presets.yaml. + options_ref: + path: presets.d/presets.yaml + preset_pool: true + - key: model + label: Model + type: select + help: "See orchestration/presets.d/models.yaml." + options_ref: + path: presets.d/models.yaml + preset_pool: true + - key: benchmark + label: Benchmark + type: select + help: "See orchestration/presets.d/benchmarks.yaml." + options_ref: + path: presets.d/benchmarks.yaml + preset_pool: true + + - id: cluster + label: Cluster & Environment + fields: + - key: cluster_config + label: Cluster Profile + type: select + help: > + Cluster-specific infra config (Kueue queue/limits, PVC + storage class, node selectors, IB/RoCE) — pick the profile + matching the Cluster selected above, not a different one. + See orchestration/presets.d/cluster_config.yaml. + options_ref: + path: presets.d/cluster_config.yaml + preset_pool: true + - key: rhoai_channel + label: RHOAI Release Channel + type: select + help: > + Optional: install a specific RHOAI operator + channel/catalog build before testing. See + orchestration/presets.d/rhoai-rc.yaml. + options_ref: + path: presets.d/rhoai-rc.yaml + preset_pool: true + + - id: advanced + label: CPT / Release / External-K8s (advanced) + fields: + - key: cpt + label: CPT / Release Recipe + type: select + help: > + Pre-composed CPT/release-testing recipe (bundles deployment + profile + benchmark + model + reporting labels). Leave unset + for an ad-hoc run using the Core fields above instead. See + orchestration/presets.d/cpt.yaml. + options_ref: + path: presets.d/cpt.yaml + preset_pool: true + - key: xks + label: External-K8s Recipe + type: select + help: > + Pre-composed recipe for external/non-OCP Kubernetes clusters + (EKS, etc). See orchestration/presets.d/xks.yaml. + options_ref: + path: presets.d/xks.yaml + preset_pool: true diff --git a/projects/rhaiis/ui/submit.yaml b/projects/rhaiis/ui/submit.yaml new file mode 100644 index 000000000..25b5bce2d --- /dev/null +++ b/projects/rhaiis/ui/submit.yaml @@ -0,0 +1,232 @@ +schema_version: 1 +project: rhaiis +title: RHAIIS +description: > + RHAIIS inference-server performance testing (vLLM / SGLang / TRT-LLM). + Field-for-field port of fournos-ui's former + app/projects/rhaiis.py + templates/projects/rhaiis_submit.html. + Accelerator, engine, model, and workload are all named presets mixed + together in presets.d, self-tagged by which key each entry sets — hence + the shared `presets_ref` pool instead of per-field `options_ref` files + for those four (see docs/ui-schema-spec.md in psap-control-center). + +modes: + - id: single + label: Single Job + default: true + presets_ref: + dir: presets.d + sections: + - id: infra + label: Infrastructure + fields: + - key: accelerator + label: Accelerator + type: radio + required: true + maps_to: rhaiis.accelerator + # trtllm only ships an Nvidia image (see + # orchestration/config.d/rhaiis.yaml, engines.trtllm.images) — + # restrict_if drops amd from this field's own options whenever + # engine == trtllm. If accelerator was already set to amd, the + # Control Center clears it so a stale, invalid combination is + # never submitted. + restrict_if: + - when: {field: engine, equals: trtllm} + exclude_values: [amd] + - key: engine + label: Engine + type: radio + required: true + maps_to: rhaiis.engine + # Bidirectional with accelerator's own restrict_if above: amd + # has no trtllm image either way, so picking amd here should + # just as much drop trtllm from Engine's options as picking + # trtllm drops amd from Accelerator's. + restrict_if: + - when: {field: accelerator, equals: amd} + exclude_values: [trtllm] + - key: engine_version + label: Engine Version + type: text + placeholder: e.g. v0.24.0 or a full image ref + help: > + Overrides the selected engine's container image for the + selected accelerator (rhaiis.engines..images.). + Leave blank to use the engine's default image. + visible_if: {field: engine, one_of: [vllm, sglang, trtllm]} + maps_to: "rhaiis.engines.{engine}.images.{accelerator}" + # NOTE: presets.d also has a `cluster_tag` preset (hera/zeus) + # used only to label dashboard/CSV rows and pick + # image_pull_secrets/fs_group — it has no effect on which + # Kubernetes cluster the job actually runs on (that's the real + # "Cluster" field, from the Control Center's own registered + # clusters, always rendered by the surrounding form). Excluded + # here on purpose to avoid a confusing double "cluster" prompt. + + - id: model + label: Model & Workload + fields: + - key: model + label: Model + type: select + required: true + maps_to: tests.rhaiis.model_key + options_ref: + path: config.d/models.yaml + label_field: name + join_key: tests.rhaiis.model_key + - key: workload + label: Workload Profiles + type: multiselect + maps_to: tests.rhaiis.workload_key + options_ref: + path: config.d/workloads.yaml + join_key: tests.rhaiis.workload_key + + - id: run_settings + label: Run Settings + fields: + - key: benchmark + label: Benchmark + type: boolean + default: true + maps_to: tests.rhaiis.run_benchmark + help: Run the benchmark workload. + - key: warmup + label: Warmup + type: boolean + default: false + maps_to: tests.rhaiis.warmup + help: Run warmup requests before the benchmark. + - key: profiler + label: Profiler + type: boolean + default: false + maps_to: rhaiis.profiler.enabled + help: Collect PyTorch profiler traces. + - key: slack + label: Slack Notifications + type: boolean + default: true + maps_to: tests.rhaiis.slack_notify_always + help: Send Slack notifications on completion. + - key: agent_analysis + label: Agent Analysis + type: boolean + default: false + maps_to: rhaiis.agent_analysis.enabled + help: > + Run AI agent analysis on results — uses tokens. Requires a + Compare Version below, and pytorch profiles for that version + on the staging dashboard. + - key: compare_versions + label: Compare Versions + type: boolean + default: false + maps_to: rhaiis.compare_versions.enabled + help: Compare results against a previous version. + - key: csv_dashboard + label: Push to Staging Dashboard + type: boolean + default: false + maps_to: caliper.postprocess.csv_dashboard.enabled + help: Push results to the staging dashboard. + - key: slack_member_id + label: Slack Member ID + type: text + placeholder: e.g. U0123456789 + help: > + In Slack: click your profile photo → Profile → the ⋮ menu → + Copy member ID. + visible_if: {field: slack, equals: true} + maps_to: tests.rhaiis.slack_member_id + - key: compare_version + label: Compare Version + type: text + placeholder: e.g. vLLM-0.23.0-benchmark + help: > + Version label to compare results against. Required when + Agent Analysis is enabled. + visible_if: {field: compare_versions, equals: true} + maps_to: tests.rhaiis.compare_version + - key: version + label: Version Label + type: text + placeholder: e.g. vLLM-0.24.0-benchmark + help: A label for this run. + maps_to: tests.rhaiis.version + + - id: cpt + label: CPT Pipeline + kind: matrix + presets_ref: + dir: presets.d + matrix: + marker_key: __cpt + models_key: __models + workloads_key: __workloads + label_key: __description + tp_key: __tp + dimensions: [model, workload] + sections: + - id: infra + label: Infrastructure + fields: + - key: accelerator + label: Accelerator + type: radio + required: true + maps_to: rhaiis.accelerator + restrict_if: + - when: {field: engine, equals: trtllm} + exclude_values: [amd] + - key: engine + label: Engine + type: radio + required: true + maps_to: rhaiis.engine + restrict_if: + - when: {field: accelerator, equals: amd} + exclude_values: [trtllm] + - key: engine_version + label: Engine Version + type: text + placeholder: e.g. v0.24.0 or a full image ref + help: > + Overrides the selected engine's container image for the + selected accelerator, across every job in this pipeline. + Leave blank to use the engine's default image. + visible_if: {field: engine, one_of: [vllm, sglang, trtllm]} + maps_to: "rhaiis.engines.{engine}.images.{accelerator}" + + - id: version + label: Version & Notifications + fields: + - key: version + label: Version Label + type: text + placeholder: e.g. vLLM-0.24.0-CPT + help: Label applied to every job in this pipeline. + maps_to: tests.rhaiis.version + - key: compare_version + label: Compare Version + type: text + placeholder: e.g. vLLM-0.23.0-benchmark + help: Version label to compare results against (optional). + maps_to: tests.rhaiis.compare_version + - key: slack_member_id + label: Slack Notification Target + type: text + placeholder: e.g. U0123456789 or S0123456789 + help: > + Slack Member ID or User Group ID (optional). For a member: + profile photo → Profile → ⋮ → Copy member ID. For a group, + use the group's own ID (starts with S). + maps_to: tests.rhaiis.slack_member_id + # NOTE: no presets.d entries are currently tagged `__cpt: true` in this + # repo, so this mode resolves to zero pipelines today. Once RHAIIS + # publishes real `__cpt` pipeline presets (the old fournos-ui + # convention: __cpt / __models / __workloads / __description / __tp — + # see git history of app/projects/rhaiis.py::fetch_config), they show + # up here automatically. No Control Center change required. From 3f0f5804b1400a4e54e1e4996f7ede261700087b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 18:14:37 +0200 Subject: [PATCH 0842/1072] [caliper] postprocess: helpers: visualization_utils: reduce the verbosity --- projects/caliper/postprocess/helpers/visualization_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/caliper/postprocess/helpers/visualization_utils.py b/projects/caliper/postprocess/helpers/visualization_utils.py index b49e5a449..4531a8627 100644 --- a/projects/caliper/postprocess/helpers/visualization_utils.py +++ b/projects/caliper/postprocess/helpers/visualization_utils.py @@ -74,7 +74,7 @@ def save_figure( output_file = output_dir / f"{final_filename}.png" try: fig.write_image(output_file, width=width, height=height) - logger.info(f"{final_filename} PNG saved successfully") + logger.debug(f"{final_filename} PNG saved successfully") return str(output_file) except Exception as png_error: # PNG generation failed - check if it's a Kaleido/Chrome issue @@ -90,7 +90,7 @@ def save_figure( logger.error(f"PNG generation failed for {final_filename}: {png_error}") return None else: - logger.info(f"Saving {final_filename} as full-page interactive HTML...") + logger.debug(f"Saving {final_filename} as full-page interactive HTML...") output_file = output_dir / f"{final_filename}.html" # Configure figure for full-screen responsive behavior @@ -100,7 +100,7 @@ def save_figure( height=None, # Remove any fixed height ) fig.write_html(output_file) - logger.info(f"{final_filename} HTML saved successfully") + logger.debug(f"{final_filename} HTML saved successfully") return str(output_file) except Exception as e: From 4f220d0b56b1ffccce8891e302207a6b0cd9a198 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 13:40:23 +0200 Subject: [PATCH 0843/1072] [guidellm] postprocess: guidellm: start using jinja --- .../guidellm/plotting/performance_analysis.py | 1206 +++-------------- .../templates/performance_report.html | 281 ++++ .../guidellm/postprocess/guidellm/plugin.py | 15 +- 3 files changed, 511 insertions(+), 991 deletions(-) create mode 100644 projects/guidellm/postprocess/guidellm/plotting/templates/performance_report.html diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index cc198a1e4..2d27ce57c 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -8,6 +8,7 @@ from typing import Any import pandas as pd +from jinja2 import Environment, FileSystemLoader from projects.caliper.engine.model import UnifiedResultRecord from projects.caliper.engine.parameter_matrix import ( @@ -874,26 +875,29 @@ def generate_token_throughput_percentiles_analysis( ) -def generate_deployment_profile_report( +def _generate_shared_performance_report( records: list[UnifiedResultRecord], output_dir: Path, title_context: str = "", report_number: int | None = None, - report_title: str = "GuideLLM Deployment Profile Analysis", + report_title: str = "GuideLLM Performance Analysis", + report_type: str = "comprehensive", # "comprehensive" or "comparison" + grouping_config: dict[str, Any] | None = None, ) -> str | None: """ - Generate performance analysis report with separate plots for comparison groups. - - Groups records by identical test conditions (all labels except the version key) and only - creates plots for groups that have records with different version values. - This enables comparing different versions under identical test conditions. + Shared function for generating performance reports with different grouping strategies. Args: records: List of unified result records output_dir: Directory to save files title_context: Additional context for titles - report_number: Optional report number for file naming (e.g., 0 for "Report 00:") - report_title: Title for the report (used in filename and HTML title) + report_number: Optional report number for file naming + report_title: Title for the report + report_type: Either "comprehensive" (loadshape grouping) or "comparison" (comparison grouping) + grouping_config: Configuration for grouping strategy + + Returns: + Path to generated report file """ try: # Create report identifier using core utilities @@ -902,154 +906,6 @@ def generate_deployment_profile_report( logger.info(f"\n🚀 Starting {display_title} generation...") logger.info("=" * 70) - # Use version as the comparison key to distinguish results - comparison_keys = {"version"} - - # First, analyze what keys are actually available in the data - all_keys = set() - comparison_key_values = {key: set() for key in comparison_keys} - - for record in records: - # Use distinguishing labels directly for analysis - labels = record.distinguishing_labels - - # Debug: Print version values found - if "version" in labels: - logger.debug( - f"🐛 DEBUG: Found version='{labels['version']}' in record {record.test_base_path}" - ) - else: - logger.debug( - f"🐛 DEBUG: No 'version' key found in record {record.test_base_path}, keys: {list(labels.keys())}" - ) - - all_keys.update(labels.keys()) - for key in comparison_keys: - if key in labels: - comparison_key_values[key].add(labels[key]) - - logger.info("\n📊 Data analysis for comparison report:") - logger.info(f" Total records: {len(records)}") - logger.info(f" Available label keys: {sorted(all_keys)}") - logger.info(" Comparison key availability:") - for key in comparison_keys: - values = comparison_key_values[key] - logger.info( - f" {key}: {len(values)} unique values: {sorted(values) if values else 'NOT FOUND'}" - ) - - # If version key has multiple values, use it; otherwise try deployment_profile fallback - active_comparison_keys = [] - for key in comparison_keys: - if len(comparison_key_values[key]) > 1: - active_comparison_keys.append(key) - - if not active_comparison_keys: - logger.info("⚠️ No version key with multiple values found, trying fallback keys...") - fallback_keys = ["deployment_profile", "guidellm_loadshape"] - fallback_values = {} - - for record in records: - # Use distinguishing labels directly for fallback logic - labels = record.distinguishing_labels - - for key in fallback_keys: - if key not in fallback_values: - fallback_values[key] = set() - - # Use distinguishing labels directly - value = labels.get(key, "default") - - fallback_values[key].add(value) - - logger.info(" Fallback key availability:") - for key in fallback_keys: - values = fallback_values[key] - logger.info(f" {key}: {len(values)} unique values: {sorted(values)}") - if len(values) > 1: - active_comparison_keys.append(key) - - if not active_comparison_keys: - logger.info("⚠️ No suitable version values found for comparison") - logger.info(" 📊 Generating single-dataset report...") - - # Use a placeholder comparison key set for single-group analysis - active_comparison_keys = ["dataset"] - comparison_keys = {"dataset"} - - logger.info(f" Using comparison keys: {active_comparison_keys}") - comparison_keys = set(active_comparison_keys) - - # Create one unified group for version comparison across all test configurations - core_labels = all_keys - - comparison_groups = {} - - for record in records: - # Create a group key from only core distinguishing labels (not version or config details) - group_labels = {} - for k, v in record.distinguishing_labels.items(): - if k not in comparison_keys and k in core_labels: - group_labels[k] = v - - # Convert to a sortable tuple for grouping - if no core labels, use a single group - if group_labels: - group_key = tuple(sorted(group_labels.items())) - else: - group_key = ("unified_comparison",) - - if group_key not in comparison_groups: - comparison_groups[group_key] = [] - comparison_groups[group_key].append(record) - - # Filter to only groups that have records with different comparison key values - valid_groups = {} - for group_key, group_records in comparison_groups.items(): - # Extract comparison key values for this group - comparison_values = set() - for record in group_records: - # Use distinguishing labels directly for comparison key evaluation - comp_values = [] - for key in comparison_keys: - value = record.distinguishing_labels.get(key, "unknown") - comp_values.append(value) - comparison_values.add(tuple(comp_values)) - - # Only include groups with multiple comparison key values - if len(comparison_values) > 1: - valid_groups[group_key] = group_records - - if not valid_groups: - logger.info( - f"⚠️ No comparison groups found - all records have identical values for keys: {active_comparison_keys}" - ) - logger.info(" 📊 Generating single-group report with all data...") - - # Create a single group with all records and include warning in the description - warning_message = f"No comparisons possible - all records have identical {', '.join(active_comparison_keys)} values" - single_group_key = ("all_data", warning_message) - valid_groups = {single_group_key: records} - - logger.info( - f"\n🔍 Found {len(valid_groups)} comparison groups with varying {active_comparison_keys}:" - ) - for i, (group_key, group_records) in enumerate(valid_groups.items(), 1): - # Handle special case for warning message - if len(group_key) == 2 and group_key[0] == "all_data": - group_desc = "All Available Data" - else: - group_desc = ", ".join(f"{k}={v}" for k, v in group_key) - comp_values = set() - for record in group_records: - # Use distinguishing labels directly for comparison key evaluation - comp_vals = [] - for key in comparison_keys: - value = record.distinguishing_labels.get(key, "unknown") - comp_vals.append(value) - comp_values.add(tuple(comp_vals)) - comp_desc = " vs ".join(f"{':'.join(cv)}" for cv in sorted(comp_values)) - logger.info(f" 📋 Group {i}: [{group_desc}] comparing [{comp_desc}]") - # Generate all the plots as figures plot_functions = [ ("Token Throughput vs Concurrency", create_token_throughput_vs_concurrency_plot), @@ -1059,35 +915,75 @@ def generate_deployment_profile_report( ("Latency vs Throughput", create_latency_vs_throughput_plot), ] + # Group records based on report type + if report_type == "comprehensive": + # Group by loadshape + groups = {} + for record in records: + loadshape = record.distinguishing_labels.get("guidellm_loadshape", "default") + if loadshape not in groups: + groups[loadshape] = [] + groups[loadshape].append(record) + logger.info(f"\n🔍 Found {len(groups)} loadshape(s): {', '.join(groups.keys())}") + else: + # Comparison grouping logic (simplified from original) + comparison_keys = ( + grouping_config.get("comparison_keys", {"version"}) + if grouping_config + else {"version"} + ) + groups = {} + for record in records: + # Create group key for comparison + group_labels = {} + for k, v in record.distinguishing_labels.items(): + if k not in comparison_keys: + group_labels[k] = v + group_key = ( + tuple(sorted(group_labels.items())) if group_labels else ("unified_comparison",) + ) + if group_key not in groups: + groups[group_key] = [] + groups[group_key].append(record) + logger.info(f"\n🔍 Found {len(groups)} comparison group(s)") + # Create dedicated directory for this report if report_number is not None: - report_dir_name = f"report_{report_number:02d}_comparison_analysis" + report_dir_name = f"report_{report_number:02d}_{report_title.lower().replace(' ', '_').replace(':', '').replace('-', '_')}" else: - report_dir_name = "comparison_analysis" + report_dir_name = "performance_analysis" report_dir = output_dir / report_dir_name report_dir.mkdir(exist_ok=True) logger.info(f"\n📁 Created report directory: {report_dir_name}") all_plots_data = [] + group_mapping = {} # Track what each group counter represents - # Process each comparison group separately - for group_key, group_records in valid_groups.items(): - # Create group description for display and directory naming - # Handle special case for warning message - if len(group_key) == 2 and group_key[0] == "all_data": - group_desc = "All Available Data" - group_name = "all_data" - warning_msg = group_key[1] # Extract warning message + # Process each group + for group_idx, (group_key, group_records) in enumerate(groups.items()): + if report_type == "comprehensive": + group_name = f"group_{group_idx:03d}" + group_desc = str(group_key) else: - group_desc = ", ".join(f"{k}={v}" for k, v in group_key) - group_name = "__".join(f"{k}_{sanitize_for_path(v)}" for k, v in group_key) - warning_msg = None + group_name = f"group_{group_idx:03d}" + group_desc = ( + ", ".join(f"{k}={v}" for k, v in group_key) + if group_key != ("unified_comparison",) + else "All Available Data" + ) + + # Store mapping for debugging/reference + group_mapping[group_name] = { + "group_key": group_key, + "group_desc": group_desc, + "record_count": len(group_records), + } - logger.info(f"\n📊 Processing comparison group: {group_desc}") + logger.info(f"\n📊 Processing group: {group_desc}") logger.info(f" Records: {len(group_records)}") - # Create DataFrame for this comparison group + # Create DataFrame for this group df = create_dataframe_from_records(group_records) if df.empty: logger.info(f" ⚠️ No data available for group: {group_desc}") @@ -1095,427 +991,198 @@ def generate_deployment_profile_report( logger.info(f" 📈 Generating {len(plot_functions)} plots for {group_desc}...") - # Create subdirectory for this comparison group + # Create subdirectory for this group group_dir = report_dir / group_name group_dir.mkdir(exist_ok=True) - # Extract metadata for this group - comp_values = set() - for r in group_records: - comp_tuple = tuple( - r.distinguishing_labels.get(key, "unknown") for key in comparison_keys - ) - comp_values.add(comp_tuple) - comp_desc = " vs ".join(f"{':'.join(cv)}" for cv in sorted(comp_values)) - + # Generate plots for this group group_plots = [] for i, (plot_name, plot_func) in enumerate(plot_functions, 1): logger.info( - f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for comparison group..." + f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for {group_desc}..." ) try: - # Create subtitle with group description and comparison info - if warning_msg: - # Show warning message instead of comparison info - subtitle = f"{group_desc} - ⚠️ {warning_msg}" + # Create context with group info + if report_type == "comprehensive": + # Extract model info for loadshape + model_info = "Unknown" + if group_records: + first_record = group_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + subtitle = f"Model: {model_info} | Load Shape: {group_desc}" else: - subtitle = f"Comparison Group: {group_desc} | Comparing: {comp_desc}" + # Comparison group subtitle + subtitle = f"Comparison Group: {group_desc}" - # Add original title context if provided - if title_context: - group_title_context = f"{title_context}
{subtitle}" - else: - group_title_context = f"
{subtitle}" + group_title_context = ( + f"{title_context}
{subtitle}" + if title_context + else f"
{subtitle}" + ) fig = plot_func(df, group_title_context) if fig: - # Save as both PNG and HTML in the group directory - filename = f"{group_name}_{plot_name.lower().replace(' ', '_')}" + # Use counter-based filename to avoid length issues + plot_name_safe = plot_name.lower().replace(" ", "_").replace("-", "_") + filename = f"{i - 1:03d}__{plot_name_safe}" - # Save PNG image config = PLOT_CONFIG_LARGE if "Percentiles" in plot_name else PLOT_CONFIG width = config["width"] height = config["height"] png_path = save_figure( fig, group_dir, filename, as_image=True, width=width, height=height ) - - # Save HTML version html_path = save_figure(fig, group_dir, filename, as_image=False) if html_path: - # Store relative paths for linking (PNG optional) png_rel_path = ( f"{report_dir_name}/{group_name}/{Path(png_path).name}" if png_path else None ) - group_plots.append( - ( - plot_name, - png_rel_path, # PNG path (can be None) - f"{report_dir_name}/{group_name}/{Path(html_path).name}", # HTML path - ) + html_rel_path = f"{report_dir_name}/{group_name}/{Path(html_path).name}" + group_plots.append((plot_name, png_rel_path, html_rel_path)) + logger.info( + f" ✅ {plot_name} saved - PNG: {png_rel_path}, HTML: {html_rel_path}" ) - logger.info(f" ✅ {plot_name} saved for comparison group") if not png_path: logger.warning( f" ⚠️ PNG version of {plot_name} could not be generated (HTML available)" ) + else: + logger.warning(f" ⚠️ HTML version of {plot_name} could not be saved") else: - logger.info( - f" ⚠️ {plot_name} could not be created for comparison group (no figure returned)" + logger.warning( + f" ⚠️ {plot_name} could not be created for {group_desc} (no figure returned)" ) except Exception as e: - logger.info(f" ❌ Failed to generate {plot_name} for comparison group: {e}") + logger.info(f" ❌ Failed to generate {plot_name} for {group_desc}: {e}") - # Store group plots data if any plots were created + # Store group plots data if group_plots: - # Include group_key for displaying identical labels - group_labels_dict = dict(group_key) if group_key != ("unified_comparison",) else {} + group_labels_dict = ( + dict(group_key) + if group_key != ("unified_comparison",) and report_type == "comparison" + else {} + ) all_plots_data.append((group_desc, group_plots, group_labels_dict)) if not all_plots_data: logger.info("❌ No plots were successfully generated") return None - logger.info( - f"\n✅ Successfully generated plots for {len(all_plots_data)} comparison group(s)!" - ) - - # Create comprehensive HTML report with sections for each comparison group - logger.info("\n📝 Assembling comprehensive HTML report...") - logger.info( - " 🔗 Creating report with comparison group sections and interactive HTML links..." - ) - - html_content = f""" - - - - - - {display_title} - - - - - - -

Click on the image to open the interactive full-size view of the plot.
- In the interactive view, click in the legend to hide a line, double click to see only this line.

+ f.write("Plot Naming Convention:\n") + f.write("-" * 30 + "\n") + f.write("000__token_throughput_vs_concurrency.png/html\n") + f.write("001__ttft_analysis.png/html\n") + f.write("002__token_throughput_percentiles.png/html\n") + f.write("003__throughput_scaling.png/html\n") + f.write("004__latency_vs_throughput.png/html\n") -
-

{display_title}

-

Generated performance analysis with separate visualizations for each comparison group

-
- -
-

Quick Navigation:

-
    """ - - # Add navigation links for each comparison group - for group_desc, _, _ in all_plots_data: - group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") - html_content += f'\n
  • {group_desc}
    • ' - - html_content += """ -
-
-""" - # Add sections for each comparison group - for group_desc, plots, group_labels in all_plots_data: - group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") - - # Handle special warning case - if group_desc == "All Available Data": - title_prefix = "Dataset:" - explanation = ( - "⚠️ No comparisons possible - all records have identical version values" - ) - else: - title_prefix = "Comparison Group:" - explanation = f"Comparing different values of {active_comparison_keys} across identical test conditions" - - # Generate display text for all identical labels - if group_labels: - labels_html = ( - "
    " - + "".join( - f"
  • {k}: {v}
    • " - for k, v in sorted(group_labels.items()) - ) - + "
" - ) - else: - labels_html = "

All data (no grouping constraints)

" - - html_content += f""" -
-

{title_prefix} {group_desc}

-

- {explanation} -

- -
- Identical labels: - {labels_html} -
- - """ - - # Embed all plots for this comparison group - if plots: - # Define plot descriptions - descriptions = { - "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", - "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", - "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", - "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", - "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", - } - - for plot_name, png_path, html_path in plots: - description = descriptions.get(plot_name, f"{plot_name} performance analysis.") - # Use centralized embedding function - plot_content = embed_plot_for_report(png_path, html_path, plot_name, output_dir) - html_content += f""" -
-

📊 {plot_name}

-

{description}

- {plot_content} -
""" + logger.info(f"📋 Group mapping saved to: {mapping_file.name}") - html_content += "
" - else: - html_content += """ -
-

⚠️ No plots available for this group.

-
-

""" - - # Close the HTML after all groups are processed - html_content += """ - -""" + # Generate HTML report + html_content = _generate_html_report( + all_plots_data, display_title, title_context, report_type, output_dir + ) # Save the main HTML report main_html_filename = create_report_filename( - "comparison_analysis", report_number, report_title, "html" + report_title.lower().replace(" ", "_"), report_number, report_title, "html" ) main_html_path = output_dir / main_html_filename with open(main_html_path, "w", encoding="utf-8") as f: f.write(html_content) - logger.info(f"✅ Comparison analysis report saved as: {main_html_filename}") + logger.info(f"✅ Report saved as: {main_html_filename}") logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") - logger.info("=" * 70) logger.info(f"🎉 {display_title} ready: {main_html_path.name}") return str(main_html_path) except Exception: - logger.exception("❌ Failed to generate deployment profile report") - # Re-raise for debugging - this error should not be silently ignored + logger.exception(f"❌ Failed to generate {report_type} performance report") raise -def generate_comprehensive_performance_report( +def _generate_html_report( + all_plots_data: list[tuple[str, list[tuple[str, str, str]], dict]], + display_title: str, + title_context: str, + report_type: str, + output_dir: Path, +) -> str: + """Generate HTML content for the performance report using Jinja2 templating.""" + + if report_type == "comprehensive": + page_description = ( + "Generated performance analysis with separate visualizations for each loadshape" + ) + nav_header = "Quick Navigation:" + else: + page_description = ( + "Generated performance analysis with separate visualizations for each comparison group" + ) + nav_header = "Quick Navigation:" + + # Set up Jinja2 environment with file-based template loading + templates_dir = Path(__file__).parent / "templates" + env = Environment(loader=FileSystemLoader(templates_dir)) + template = env.get_template("performance_report.html") + + # Render template with context + html_content = template.render( + display_title=display_title, + page_description=page_description, + nav_header=nav_header, + all_plots_data=all_plots_data, + report_type=report_type, + output_dir=output_dir, + embed_plot_for_report=embed_plot_for_report, + ) + + return html_content + + +def generate_deployment_profile_report( records: list[UnifiedResultRecord], output_dir: Path, title_context: str = "", report_number: int | None = None, - report_title: str = "GuideLLM Performance Analysis", + report_title: str = "GuideLLM Deployment Profile Analysis", ) -> str | None: """ - Generate comprehensive performance analysis report with separate plots for each loadshape. + Generate performance analysis report with separate plots for comparison groups. + + Groups records by identical test conditions (all labels except the version key) and only + creates plots for groups that have records with different version values. + This enables comparing different versions under identical test conditions. Args: records: List of unified result records @@ -1524,475 +1191,42 @@ def generate_comprehensive_performance_report( report_number: Optional report number for file naming (e.g., 0 for "Report 00:") report_title: Title for the report (used in filename and HTML title) """ - try: - # Create report identifier using core utilities - display_title = create_report_title_display(report_title, report_number) - - logger.info(f"\n🚀 Starting {display_title} generation...") - logger.info("=" * 70) - - # Group records by guidellm_loadshape - loadshape_groups = {} - for record in records: - loadshape = record.distinguishing_labels.get("guidellm_loadshape", "default") - if loadshape not in loadshape_groups: - loadshape_groups[loadshape] = [] - loadshape_groups[loadshape].append(record) - - if not loadshape_groups: - logger.info("❌ No data available for analysis") - return None - - logger.info( - f"\n🔍 Found {len(loadshape_groups)} loadshape(s): {', '.join(loadshape_groups.keys())}" - ) - - # Generate all the plots as figures - plot_functions = [ - ("Token Throughput vs Concurrency", create_token_throughput_vs_concurrency_plot), - ("TTFT Analysis", create_ttft_analysis_plot), - ("Token Throughput Percentiles", create_token_throughput_percentiles_plot), - ("Throughput Scaling", create_throughput_scaling_plot), - ("Latency vs Throughput", create_latency_vs_throughput_plot), - ] - - # Create dedicated directory for this report - if report_number is not None: - report_dir_name = f"report_{report_number:02d}_{report_title.lower().replace(' ', '_').replace(':', '').replace('-', '_')}" - else: - report_dir_name = "performance_analysis" - - report_dir = output_dir / report_dir_name - report_dir.mkdir(exist_ok=True) - logger.info(f"\n📁 Created report directory: {report_dir_name}") - - all_plots_data = [] - - # Process each loadshape separately - for loadshape, loadshape_records in loadshape_groups.items(): - logger.info(f"\n📊 Processing loadshape: {loadshape}") - logger.info(f" Records: {len(loadshape_records)}") - - # Create DataFrame for this loadshape - df = create_dataframe_from_records(loadshape_records) - if df.empty: - logger.info(f" ⚠️ No data available for loadshape: {loadshape}") - continue - - logger.info(f" 📈 Generating {len(plot_functions)} plots for {loadshape}...") - - # Create subdirectory for this loadshape - loadshape_dir = report_dir / sanitize_for_path(loadshape) - loadshape_dir.mkdir(exist_ok=True) - - # Extract model information for this loadshape - model_info = "Unknown" - if loadshape_records: - # Try to get model information from distinguishing labels - first_record = loadshape_records[0] - model_info = ( - first_record.distinguishing_labels.get("model") - or first_record.distinguishing_labels.get("model_name") - or first_record.distinguishing_labels.get("llm_model") - or first_record.run_identity.get("model") - or "Unknown" - ) - - loadshape_plots = [] - for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - logger.info( - f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for {loadshape}..." - ) - try: - # Create subtitle with model and loadshape info - subtitle = f"Model: {model_info} | Load Shape: {loadshape}" - - # Add original title context if provided - if title_context: - loadshape_title_context = f"{title_context}
{subtitle}" - else: - loadshape_title_context = f"
{subtitle}" - - fig = plot_func(df, loadshape_title_context) - if fig: - # Save as both PNG and HTML in the loadshape directory - filename = f"{loadshape}_{plot_name.lower().replace(' ', '_')}" - - # Save PNG image - config = PLOT_CONFIG_LARGE if "Percentiles" in plot_name else PLOT_CONFIG - width = config["width"] - height = config["height"] - png_path = save_figure( - fig, loadshape_dir, filename, as_image=True, width=width, height=height - ) - - # Save HTML version - html_path = save_figure(fig, loadshape_dir, filename, as_image=False) - - if html_path: - # Store relative paths for linking (PNG optional) - png_rel_path = ( - f"{report_dir_name}/{loadshape}/{Path(png_path).name}" - if png_path - else None - ) - html_rel_path = f"{report_dir_name}/{loadshape}/{Path(html_path).name}" - loadshape_plots.append( - ( - plot_name, - png_rel_path, # PNG path (can be None) - html_rel_path, # HTML path - ) - ) - logger.info( - f" ✅ {plot_name} saved - PNG: {png_rel_path}, HTML: {html_rel_path}" - ) - if not png_path: - logger.warning( - f" ⚠️ PNG version of {plot_name} could not be generated (HTML available)" - ) - else: - logger.warning(f" ⚠️ HTML version of {plot_name} could not be saved") - else: - logger.warning( - f" ⚠️ {plot_name} could not be created for {loadshape} (no figure returned)" - ) - - except Exception as e: - logger.info(f" ❌ Failed to generate {plot_name} for {loadshape}: {e}") - - # Store loadshape plots data - if loadshape_plots: - all_plots_data.append((loadshape, loadshape_plots)) - - if not all_plots_data: - logger.info("❌ No plots were successfully generated") - return None - - logger.info(f"\n✅ Successfully generated plots for {len(all_plots_data)} loadshape(s)!") - - # Create comprehensive HTML report with sections for each loadshape - logger.info("\n📝 Assembling comprehensive HTML report...") - logger.info(" 🔗 Creating report with loadshape sections and interactive HTML links...") - - html_content = f""" - - - - - - {display_title} - - - - - - -

Click on the image to open the interactive full-size view of the plot.
- In the interactive view, click in the legend to hide a line, double click to see only this line.

- -
-

{display_title}

-

Generated performance analysis with separate visualizations for each loadshape

-
- -
-

Quick Navigation:

-
    """ - - # Add navigation links for each loadshape - for loadshape, _ in all_plots_data: - html_content += f'\n
  • {loadshape}
    • ' - - html_content += """ -
-
-""" - - # Add sections for each loadshape with tabbed interface - container_id = 0 - for loadshape, plots in all_plots_data: - # Extract model information for this loadshape - loadshape_records = loadshape_groups[loadshape] - model_info = "Unknown" - if loadshape_records: - # Try to get model information from distinguishing labels - first_record = loadshape_records[0] - model_info = ( - first_record.distinguishing_labels.get("model") - or first_record.distinguishing_labels.get("model_name") - or first_record.distinguishing_labels.get("llm_model") - or first_record.run_identity.get("model") - or "Unknown" - ) - - html_content += f""" -
-

Loadshape: {loadshape}

-

- Model: {model_info} | Load Shape: {loadshape} -

- -
-
""" - - # Define tab mapping with icons - tab_mapping = { - "Token Throughput vs Concurrency": "🚀 Throughput", - "TTFT Analysis": "⏱️ TTFT", - "Token Throughput Percentiles": "📊 Throughput Percentiles", - "Throughput Scaling": "📈 Scaling", - "Latency vs Throughput": "⚖️ Latency Trade-off", - } - - # Create tab headers - for tab_idx, (plot_name, _png_path, _html_path) in enumerate(plots): - tab_title = tab_mapping.get(plot_name, plot_name) - active_class = " active" if tab_idx == 0 else "" - html_content += f""" - """ - - html_content += """ -
""" - - # Create tab contents - descriptions = { - "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", - "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", - "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", - "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", - "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", - } - - for tab_idx, (plot_name, png_path, html_path) in enumerate(plots): - active_class = " active" if tab_idx == 0 else "" - description = descriptions.get(plot_name, f"{plot_name} performance analysis.") - # Use centralized embedding function - plot_content = embed_plot_for_report(png_path, html_path, plot_name, output_dir) - - html_content += f""" -
-
-

{plot_name}

-

{description}

- {plot_content} -
-
""" - - html_content += """ -
-
""" - container_id += 1 - - html_content += """ - -""" - - # Save the main HTML report - main_html_filename = create_report_filename( - report_title.lower().replace(" ", "_"), report_number, report_title, "html" - ) - main_html_path = output_dir / main_html_filename - - with open(main_html_path, "w", encoding="utf-8") as f: - f.write(html_content) - - logger.info(f"✅ Comprehensive report saved as: {main_html_filename}") - logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") + return _generate_shared_performance_report( + records=records, + output_dir=output_dir, + title_context=title_context, + report_number=report_number, + report_title=report_title, + report_type="comparison", + grouping_config={"comparison_keys": {"version"}}, + ) - logger.info("=" * 70) - logger.info(f"🎉 {display_title} ready: {main_html_path.name}") - return str(main_html_path) +def generate_comprehensive_performance_report( + records: list[UnifiedResultRecord], + output_dir: Path, + title_context: str = "", + report_number: int | None = None, + report_title: str = "GuideLLM Performance Analysis", +) -> str | None: + """ + Generate comprehensive performance analysis report with separate plots for each loadshape. - except Exception: - logger.exception("❌ Failed to generate comprehensive performance report") - # Re-raise for debugging - this error should not be silently ignored - raise + Args: + records: List of unified result records + output_dir: Directory to save files + title_context: Additional context for titles + report_number: Optional report number for file naming (e.g., 0 for "Report 00:") + report_title: Title for the report (used in filename and HTML title) + """ + return _generate_shared_performance_report( + records=records, + output_dir=output_dir, + title_context=title_context, + report_number=report_number, + report_title=report_title, + report_type="comprehensive", + ) def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: diff --git a/projects/guidellm/postprocess/guidellm/plotting/templates/performance_report.html b/projects/guidellm/postprocess/guidellm/plotting/templates/performance_report.html new file mode 100644 index 000000000..c6cb5d200 --- /dev/null +++ b/projects/guidellm/postprocess/guidellm/plotting/templates/performance_report.html @@ -0,0 +1,281 @@ + + + + + + {{ display_title }} + + + + + + +

Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.

+ +
+

{{ display_title }}

+

{{ page_description }}

+
+ +
+

{{ nav_header }}

+
    + {%- for group_desc, _, _ in all_plots_data %} + {%- set group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") %} +
  • {{ group_desc }}
    • + {%- endfor %} +
+
+ + {%- set container_id = [0] %} + {%- for group_desc, plots, group_labels in all_plots_data %} + {%- set group_id = group_desc.replace(" ", "_").replace("=", "_").replace(",", "_") %} +
+

{{ report_type.title() }}: {{ group_desc }}

+ + {%- if report_type == "comprehensive" %} +
+
+ {%- set tab_mapping = { + "Token Throughput vs Concurrency": "🚀 Throughput", + "TTFT Analysis": "⏱️ TTFT", + "Token Throughput Percentiles": "📊 Throughput Percentiles", + "Throughput Scaling": "📈 Scaling", + "Latency vs Throughput": "⚖️ Latency Trade-off", + } %} + {%- for plot_name, png_path, html_path in plots %} + {%- set tab_title = tab_mapping.get(plot_name, plot_name) %} + {%- set active_class = " active" if loop.index0 == 0 else "" %} + + {%- endfor %} +
+ + {%- set descriptions = { + "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", + "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", + "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", + "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", + "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", + } %} + + {%- for plot_name, png_path, html_path in plots %} + {%- set active_class = " active" if loop.index0 == 0 else "" %} + {%- set description = descriptions.get(plot_name, plot_name + " performance analysis.") %} +
+
+

{{ plot_name }}

+

{{ description }}

+ {{ embed_plot_for_report(png_path, html_path, plot_name, output_dir) }} +
+
+ {%- endfor %} +
+ {%- set _ = container_id.append(container_id.pop() + 1) %} + + {%- else %} + {%- set descriptions = { + "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", + "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", + "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", + "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", + "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", + } %} + {%- for plot_name, png_path, html_path in plots %} + {%- set description = descriptions.get(plot_name, plot_name + " performance analysis.") %} +
+

📊 {{ plot_name }}

+

{{ description }}

+ {{ embed_plot_for_report(png_path, html_path, plot_name, output_dir) }} +
+ {%- endfor %} + {%- endif %} +
+ {%- endfor %} + + + \ No newline at end of file diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index 799ab03c1..8a2b8a074 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -42,7 +42,6 @@ def _ensure_loaded(self) -> dict[str, dict[str, Any]]: "function": generate_comprehensive_performance_report, "type": "report", "kwargs": { - "report_number": 0, "report_title": "GuideLLM Performance Analysis", }, "description": "comprehensive performance analysis report (recommended)", @@ -51,7 +50,6 @@ def _ensure_loaded(self) -> dict[str, dict[str, Any]]: "function": generate_deployment_profile_report, "type": "report", "kwargs": { - "report_number": 2, "report_title": "GuideLLM Deployment Profile Analysis", }, "description": "performance analysis comparing different product versions/models under identical test conditions", @@ -156,7 +154,7 @@ def register_plot( # noqa: FBT001 my_custom_function, "My custom analysis", type_="plot", - report_number=10 + custom_param="value" ) """ PLOT_REGISTRY[name] = { @@ -177,7 +175,8 @@ def visualize( """Generate visualization reports for GuideLLM benchmarks.""" output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] - wanted = frozenset(report_ids or ()) + # Preserve caller-supplied order while removing duplicates + wanted = list(dict.fromkeys(report_ids or ())) # Filter to only GuideLLM records with benchmarks guidellm_records = [ @@ -191,6 +190,7 @@ def visualize( # Generate reports using the registry invalid_reports: list[str] = [] + report_counter = 0 for report_name in wanted: if report_name not in PLOT_REGISTRY: @@ -200,7 +200,12 @@ def visualize( plot_config = PLOT_REGISTRY[report_name] function = plot_config["function"] - kwargs = plot_config.get("kwargs", {}) + kwargs = plot_config.get("kwargs", {}).copy() # Copy to avoid modifying the registry + + # Add automatic report numbering for report types + if plot_config["type"] == "report": + kwargs["report_number"] = report_counter + report_counter += 1 try: # Call the generator function with records, output_dir, and any additional kwargs From e50760648e34a786c78ad96f253eb1ff3073a6eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 14:02:45 +0200 Subject: [PATCH 0844/1072] [guidellm] postprocess: guidellm: parsing/parsers: don't extract the model_name from the llmisvc --- .../guidellm/postprocess/guidellm/parsing/parsers.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index bc7bc34b7..62e4a988e 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -153,7 +153,7 @@ def extract_fields_from_llmisvc(self, file_path: Path) -> dict[str, Any]: file_path: Path to llminferenceservice.yaml file Returns: - Dictionary with extracted fields (product_version, deployment_profile, model_name) + Dictionary with extracted fields (product_version, deployment_profile) """ result = {} try: @@ -193,12 +193,6 @@ def extract_fields_from_llmisvc(self, file_path: Path) -> dict[str, Any]: f"Extracted deployment_profile '{deployment_profile}' from {file_path}" ) - # Extract model name from spec - model_name = extract_field_by_jsonpath(yaml_data, "spec.model.name") - if model_name: - result["model_name"] = model_name - logger.debug(f"Extracted model_name '{model_name}' from {file_path}") - replicas = extract_field_by_jsonpath(yaml_data, "spec.replicas") if replicas is not None: result["replicas"] = replicas @@ -717,7 +711,6 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: "gpu_type", "product_version", "deployment_profile", - "model_name", "cluster", "benchmark_key", ] From d3fd92b39f67704626f607fa9fc2dd416efa47b3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 14:03:02 +0200 Subject: [PATCH 0845/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: reduce the verbosity --- .../guidellm/plotting/performance_analysis.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 2d27ce57c..bca4a4005 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -68,7 +68,7 @@ def _read_html_content(html_path: str | Path) -> str: HTML content string """ try: - logger.info(f"🔍 Reading HTML content from: {html_path}") + logger.debug(f"🔍 Reading HTML content from: {html_path}") # Check if file exists if not Path(html_path).exists(): @@ -78,7 +78,7 @@ def _read_html_content(html_path: str | Path) -> str: with open(html_path, encoding="utf-8") as html_file: content = html_file.read() content_size_kb = len(content.encode("utf-8")) / 1024 - logger.info(f" 📄 Read {content_size_kb:.1f} KB from HTML file") + logger.debug(f" 📄 Read {content_size_kb:.1f} KB from HTML file") # Import re for regex operations import re @@ -88,12 +88,12 @@ def _read_html_content(html_path: str | Path) -> str: if body_match: body_content = body_match.group(1) body_size_kb = len(body_content.encode("utf-8")) / 1024 - logger.info(f" ✅ Extracted body content: {body_size_kb:.1f} KB") + logger.debug(f" ✅ Extracted body content: {body_size_kb:.1f} KB") return body_content else: # If no body tags found, use full content - logger.warning(" ⚠️ No body tags found, using full content") - logger.info(f" 📄 Using full HTML content: {content_size_kb:.1f} KB") + logger.debug(" ⚠️ No body tags found, using full content") + logger.debug(f" 📄 Using full HTML content: {content_size_kb:.1f} KB") return content except Exception as e: logger.warning(f"❌ Failed to read HTML content from {html_path}: {e}") @@ -160,20 +160,20 @@ def embed_plot_for_report( Returns: HTML string with embedded plot content """ - logger.info(f"🔗 Embedding plot: {plot_name}") - logger.info(f" 📁 Output dir: {output_dir}") - logger.info(f" 🌐 HTML path: {html_path}") - logger.info(f" 🖼️ PNG path: {png_path}") + logger.debug(f"🔗 Embedding plot: {plot_name}") + logger.debug(f" 📁 Output dir: {output_dir}") + logger.debug(f" 🌐 HTML path: {html_path}") + logger.debug(f" 🖼️ PNG path: {png_path}") # Resolve full HTML path full_html_path = Path(output_dir) / html_path - logger.info(f" 📄 Full HTML path: {full_html_path}") + logger.debug(f" 📄 Full HTML path: {full_html_path}") # Read HTML content for direct embedding plot_html_content = _read_html_content(full_html_path) if not plot_html_content.strip(): - logger.warning(f" ⚠️ Empty HTML content for {plot_name}") + logger.debug(f" ⚠️ Empty HTML content for {plot_name}") return f"

⚠️ Could not load interactive plot for {plot_name}

" # Create plot with optional PNG spoiler @@ -182,7 +182,7 @@ def embed_plot_for_report( ) result_size_kb = len(result.encode("utf-8")) / 1024 - logger.info(f" ✅ Successfully embedded {plot_name} ({result_size_kb:.1f} KB)") + logger.debug(f" ✅ Successfully embedded {plot_name} ({result_size_kb:.1f} KB)") return result From 0334f3e97a40283023e92db08b91a8771d71c0bf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 18:15:07 +0200 Subject: [PATCH 0846/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: reduce the verbosity --- .../guidellm/plotting/performance_analysis.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index bca4a4005..46f6bbf49 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -620,10 +620,10 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): ttft_data = df["ttft_median_ms"] non_zero_count = (ttft_data > 0).sum() - logger.info( + logger.debug( f" TTFT data: {len(ttft_data)} total points, {non_zero_count} non-zero values" ) - logger.info(f" TTFT range: {ttft_data.min():.1f} - {ttft_data.max():.1f} ms") + logger.debug(f" TTFT range: {ttft_data.min():.1f} - {ttft_data.max():.1f} ms") if ttft_data.max() == 0: logger.warning("⚠️ All TTFT values are zero - plot may appear empty") @@ -672,7 +672,7 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput percentiles plot.""" try: - logger.info("📈 Creating token throughput percentiles distribution plot...") + logger.debug("📈 Creating token throughput percentiles distribution plot...") import plotly.express as px import plotly.graph_objects as go @@ -717,7 +717,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st # Get unique configurations and colors - maintain custom sort order configurations = df["test_configuration"].drop_duplicates().tolist() - logger.info( + logger.debug( f" Plotting {len(configurations)} configurations with percentile distributions..." ) available_colors = px.colors.qualitative.Set1 @@ -734,7 +734,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ("P75", "output_tokens_per_second_p75", {"width": 3, "dash": "dash"}, 0.9), ("P90", "output_tokens_per_second_p90", {"width": 2, "dash": "dashdot"}, 0.8), ] - logger.info(f" Adding {len(percentiles)} percentile lines per configuration...") + logger.debug(f" Adding {len(percentiles)} percentile lines per configuration...") for config in configurations: config_df = df[df["test_configuration"] == config].sort_values("intended_concurrency") @@ -761,7 +761,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ) fig.update_yaxes(rangemode="tozero") - logger.info("✅ Token throughput percentiles plot created successfully") + logger.debug("✅ Token throughput percentiles plot created successfully") return fig except Exception as e: @@ -816,7 +816,7 @@ def generate_token_throughput_vs_concurrency( report_number: int | None = None, ) -> str | None: """Generate token throughput vs concurrency analysis and save to file.""" - logger.info("\n🚀 Generating token throughput vs concurrency analysis...") + logger.debug("\n🚀 Generating token throughput vs concurrency analysis...") df = create_dataframe_from_records(records) if df.empty: return None @@ -1048,7 +1048,7 @@ def _generate_shared_performance_report( ) html_rel_path = f"{report_dir_name}/{group_name}/{Path(html_path).name}" group_plots.append((plot_name, png_rel_path, html_rel_path)) - logger.info( + logger.debug( f" ✅ {plot_name} saved - PNG: {png_rel_path}, HTML: {html_rel_path}" ) if not png_path: From 38ff8d61a8c2afe5f658d6b9164c39c00f26967d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 18:16:33 +0200 Subject: [PATCH 0847/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: update the image size --- .../postprocess/guidellm/plotting/performance_analysis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 46f6bbf49..2f752d4fc 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -25,15 +25,15 @@ # Plot configuration constants PLOT_CONFIG = { - "width": 1700, - "height": 500, + "width": 1200, + "height": 650, "font": {"size": 12}, "title_font_size": 16, } PLOT_CONFIG_LARGE = { - "width": 1700, - "height": 600, + "width": 1200, + "height": 650, "font": {"size": 12}, "title_font_size": 16, } From 0591409e675d2b36670d653cd576fdc4a0320c30 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 18:16:51 +0200 Subject: [PATCH 0848/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: add the html link --- .../guidellm/plotting/performance_analysis.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 2f752d4fc..cf5dae815 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -125,19 +125,33 @@ def _create_plot_with_optional_png_spoiler( {html_content}
""" - # Add simple link to PNG version if it exists + # Add links to PNG and HTML versions if they exist + links = [] + if png_path is not None: full_png_path = Path(output_dir) / png_path if full_png_path.exists(): - result += f""" - """ + links.append( + f'📸 Image version' + ) else: logger.debug(f"PNG file not found for {plot_name} at {full_png_path}") else: logger.debug(f"PNG generation was skipped for {plot_name} (HTML-only mode)") + if html_path is not None: + full_html_path = Path(output_dir) / html_path + if full_html_path.exists(): + links.append( + f'🌐 HTML version' + ) + + if links: + result += f""" +
+ {" | ".join(links)} +
""" + return result From f9938c47601287c8699f6d5d69775362c08b0a99 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 14:04:20 +0200 Subject: [PATCH 0849/1072] [llm_d] orchestration: test_phase: save the platform in the KPI labels --- .../llm_d/orchestration/config.d/cpt.yaml | 2 +- .../llm_d/orchestration/presets.d/cpt.yaml | 1 + projects/llm_d/orchestration/test_phase.py | 20 ++++++++++++++++--- .../llm_d/tests/test_deployment_profiles.py | 1 + 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/cpt.yaml b/projects/llm_d/orchestration/config.d/cpt.yaml index de0fc032f..b20c54f90 100644 --- a/projects/llm_d/orchestration/config.d/cpt.yaml +++ b/projects/llm_d/orchestration/config.d/cpt.yaml @@ -2,5 +2,5 @@ kpi: labels: platform: OCP test_harness: null - product_version: null + product_version: RHOAI # placeholder gpu_type: null diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml index 02250eef8..2dcab3423 100644 --- a/projects/llm_d/orchestration/presets.d/cpt.yaml +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -16,6 +16,7 @@ cpt: caliper.postprocess.s3.instance: forge-cpt caliper.postprocess.s3.directory: llm-d/default caliper.postprocess.s3.export.enabled: true + cpt.kpi.labels.product_version: null # must be overridden when testing for CPT cpt-xks: extends: diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index 68e9fb207..d4166bab2 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -163,9 +163,23 @@ def extract_kpi_labels_from_config() -> dict[str, str]: if v is None: del kpi_labels[k] - product_version = config.project.get_config("cpt.kpi.labels.product_version") - if product_version: - kpi_labels["product_version"] = product_version + # Ensure critical labels are present (fail fast if missing) + model_name = runtime_config.get_model_name() + if not model_name: + raise ValueError("model_name is required but not configured in runtime config") + kpi_labels["model_name"] = model_name + + # Validate that essential labels are configured + essential_label_configs = [ + ("platform", "cpt.kpi.labels.platform"), + ] + + for label_name, config_path in essential_label_configs: + if label_name not in kpi_labels: # Not already set by cpt.kpi.labels + value = config.project.get_config(config_path) + if not value: + raise ValueError(f"{label_name} is required but not configured at {config_path}") + kpi_labels[label_name] = value return kpi_labels diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index e52e4603b..17197f25f 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -21,6 +21,7 @@ CONFIG_OVERRIDES = { "runtime.kserve.dry_run": True, "caliper.postprocess.enabled": False, + "cpt.kpi.labels.product_version": "RHOAI-XXX", "agentic.enabled": False, } From 65e1ecfb169c41789b18903382d09672b4cc0ff2 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 9 Sep 2026 14:04:47 +0200 Subject: [PATCH 0850/1072] [llm_d] postprocess: llm_d: parsing/kpis: don't extract the model_name from metrics.model_name --- projects/llm_d/postprocess/llm_d/parsing/kpis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/llm_d/postprocess/llm_d/parsing/kpis.py b/projects/llm_d/postprocess/llm_d/parsing/kpis.py index 2b2bf48f9..397027f81 100644 --- a/projects/llm_d/postprocess/llm_d/parsing/kpis.py +++ b/projects/llm_d/postprocess/llm_d/parsing/kpis.py @@ -33,7 +33,6 @@ def _extract_labels(record) -> dict[str, Any]: "product_version": "metrics.product_version", "cluster": "metrics.cluster", "deployment_profile": "metrics.deployment_profile", - "model_name": "metrics.model_name", "guidellm_loadshape": "metrics.benchmark_key", } From 3792ea30d82f126bca7a66c492c4fb9bb510e4b9 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 09:44:46 +0200 Subject: [PATCH 0851/1072] [llm_d] postprocess: llm_d: parsing/kpis: validate the the KPIs have the required labels --- .../llm_d/postprocess/llm_d/parsing/kpis.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/projects/llm_d/postprocess/llm_d/parsing/kpis.py b/projects/llm_d/postprocess/llm_d/parsing/kpis.py index 397027f81..bb22cfda8 100644 --- a/projects/llm_d/postprocess/llm_d/parsing/kpis.py +++ b/projects/llm_d/postprocess/llm_d/parsing/kpis.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from datetime import UTC, datetime from typing import Any @@ -15,10 +16,24 @@ ) from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class GuideLLMKpiHandler: """Handles KPI catalog and computation for GuideLLM benchmarks.""" + # Define required labels that must be present in the final label set + REQUIRED_LABELS = { + "model_name", + "product_version", + "deployment_profile", + "guidellm_loadshape", + "gpu_type", + "platform", + "test_harness", + "benchmark_key", + } + # Define custom label extractor for GuideLLM with fallbacks for missing kpi_labels @staticmethod def _extract_labels(record) -> dict[str, Any]: @@ -45,6 +60,11 @@ def _extract_labels(record) -> dict[str, Any]: return labels + @staticmethod + def _validate_required_labels(labels: dict[str, Any]) -> set[str]: + """Validate required labels and return any missing labels.""" + return GuideLLMKpiHandler.REQUIRED_LABELS - labels.keys() + LABEL_EXTRACTOR = type("TestLabelExtractor", (), {"extract": _extract_labels})() # Metadata fields to include in KPI records but not as labels @@ -121,6 +141,31 @@ def compute_kpis( status = KpiComputationStatus.failure_status(error_msg) return [], status + # Check for missing required labels in records + invalid_label_records = [] + logger.info(f"🔍 Validating required labels for {len(valid_records)} records...") + logger.info(f" Required labels: {sorted(GuideLLMKpiHandler.REQUIRED_LABELS)}") + + for r in valid_records: + test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) + missing_labels = GuideLLMKpiHandler._validate_required_labels(test_condition_labels) + + logger.debug(f" Record {r.test_base_path}:") + logger.debug(f" Available: {sorted(test_condition_labels.keys())}") + logger.debug(f" Missing: {sorted(missing_labels) if missing_labels else 'None'}") + + if missing_labels: + invalid_label_records.append( + { + "test_path": r.test_base_path, + "missing_labels": sorted(missing_labels), + "available_labels": sorted(test_condition_labels.keys()), + } + ) + logger.warning(f" ❌ {r.test_base_path}: missing {sorted(missing_labels)}") + + logger.info(f" Validation complete: {len(invalid_label_records)} invalid records found") + # Group records by test path for curve KPIs (same test, different rates) from collections import defaultdict @@ -134,6 +179,12 @@ def compute_kpis( # Generate scalar KPIs for each record for r in valid_records: test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) + + # Skip records with missing required labels + missing_labels = GuideLLMKpiHandler._validate_required_labels(test_condition_labels) + if missing_labels: + continue + metadata_fields = GuideLLMKpiHandler.extract_metadata(r) # Compute scalar KPIs only @@ -182,6 +233,12 @@ def compute_kpis( continue kpi_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) + + # Skip records with missing required labels + missing_labels = GuideLLMKpiHandler._validate_required_labels(kpi_labels) + if missing_labels: + continue + metadata_fields = GuideLLMKpiHandler.extract_metadata(r) # Generate curve KPIs from performance curves @@ -224,6 +281,27 @@ def compute_kpis( out.append(kpi_record) processed_records.add(r.test_base_path) # Track that this record produced a KPI + # Check if any records had missing required labels + if invalid_label_records: + error_msg = ( + f"Found missing required labels in {len(invalid_label_records)} test paths:\n" + + "\n".join( + [ + f" - {r['test_path']}: missing {r['missing_labels']}, available {r['available_labels']}" + for r in invalid_label_records[:3] + ] + ) + + ("..." if len(invalid_label_records) > 3 else "") + + "\nPlease ensure all required labels are set in the KPI labels configuration." + ) + status = KpiComputationStatus.failure_status(error_msg) + logger.error("❌ KPI computation failed due to missing required labels") + logger.error(f" Generated {len(out)} KPIs before failing") + return out, status + # Create success status status = KpiComputationStatus.success_status(len(processed_records), len(valid_records)) + logger.info( + f"✅ KPI computation successful: {len(out)} KPIs generated from {len(processed_records)} records" + ) return out, status From 082ada817606e82a5b7fc9035aede15177b248f0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 09:52:41 +0200 Subject: [PATCH 0852/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: correctly sort the groups --- .../postprocess/guidellm/plotting/performance_analysis.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index cf5dae815..c777c0677 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -938,7 +938,9 @@ def _generate_shared_performance_report( if loadshape not in groups: groups[loadshape] = [] groups[loadshape].append(record) - logger.info(f"\n🔍 Found {len(groups)} loadshape(s): {', '.join(groups.keys())}") + logger.info( + f"\n🔍 Found {len(groups)} loadshape(s): {', '.join(sorted(groups.keys()))}" + ) else: # Comparison grouping logic (simplified from original) comparison_keys = ( @@ -974,8 +976,8 @@ def _generate_shared_performance_report( all_plots_data = [] group_mapping = {} # Track what each group counter represents - # Process each group - for group_idx, (group_key, group_records) in enumerate(groups.items()): + # Process each group (sorted by key for consistent ordering) + for group_idx, (group_key, group_records) in enumerate(sorted(groups.items())): if report_type == "comprehensive": group_name = f"group_{group_idx:03d}" group_desc = str(group_key) From 4ad60bcf95f20a3cecfff16ad1c5c9f113dbb568 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 13:18:18 +0200 Subject: [PATCH 0853/1072] [guidellm] postprocess: guidellm: plotting/performance_analysis: optimize the embedded HTML From 2132869ba03212c2715e417b826675a9a92bc243 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 10 Sep 2026 20:25:57 +0200 Subject: [PATCH 0854/1072] [caliper] orchestration: notification: reorder the regression testing items --- projects/caliper/orchestration/notification.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index c7bfdebca..baae62a49 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -239,8 +239,7 @@ def _format_analyse_kpis_step(step_data: dict, get_file_link: callable | None) - lines.append(_create_file_link(html_file, "🌐", get_file_link)) # Show analysis output file - output_file = step_data.get("output_file") - if output_file: + if output_file := step_data.get("output_file"): lines.append(_create_file_link(output_file, "📊", get_file_link)) step_status = step_data.get("status") From 66a4f9bb56d2e47f8c03844d7c80374825894bfb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 09:21:38 +0200 Subject: [PATCH 0855/1072] [core] library: export_notifications: update the Censoring Report link --- projects/core/library/export_notifications.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 4b64fd7a9..af959d0a6 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -593,7 +593,7 @@ def _build_enhanced_notification( return "\n".join(notification_parts), notification_success -def _get_censoring_report_section(artifact_dir: Path) -> list[str] | None: +def _get_censoring_report_section(artifact_dir: Path | None) -> list[str] | None: """ Parse censoring_report.yaml if it exists and return notification section. @@ -1493,31 +1493,30 @@ def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) - safe_censored_files = report_data.get("safe_censored_files", 0) censored_files = report_data.get("censored_files", 0) # Unexpected censoring - censoring_lines.append("* 🔒 Censoring Report") # Create link to censoring report file if get_file_link: try: report_link = get_file_link(censoring_report_path) - first_line = f"[📊 {total_files} files scanned]({report_link})" + censoring_lines.append(f"* 🔒 [Censoring Report]({report_link})") except Exception as e: logger.warning( f"Failed to create link for censoring report {censoring_report_path}: {e}" ) - first_line = f"📊 {total_files} files scanned" + censoring_lines.append("* 🔒 Censoring Report") else: - first_line = f"📊 {total_files} files scanned" + censoring_lines.append("* 🔒 Censoring Report") - censoring_lines.append(f" * {first_line}") + censoring_lines.append(f" * 📊 {total_files} files scanned") # Show breakdown of file types if clean_files > 0: - censoring_lines.append(f" * ✅ Clean files: {clean_files}") + censoring_lines.append(f" * ✅ Clean files: {clean_files}") if safe_censored_files > 0: - censoring_lines.append(f" * 🔐 Safe replacements: {safe_censored_files}") + censoring_lines.append(f" * 🔐 Safe replacements: {safe_censored_files}") if censored_files > 0: - censoring_lines.append(f" * ⚠️ Unexpected censoring: {censored_files}") + censoring_lines.append(f" * ⚠️ Unexpected censoring: {censored_files}") # Show details for unexpected censoring if available censored_by_reason = report_data.get("censored_by_reason", {}) From 5bbf82d1ddaaba24aa2dc8677d6c8863f0e50e62 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 09:38:23 +0200 Subject: [PATCH 0856/1072] [core] library: export_notifications: include the total duration in the base_status --- projects/core/library/export_notifications.py | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index af959d0a6..ece0f852c 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -478,7 +478,7 @@ def _extract_finish_reason_from_status(status: ExportStatus) -> str: def _build_enhanced_notification( - artifact_dir: Path, + artifact_dir: Path | None, project: str, finish_reason: str, status: ExportStatus, @@ -506,7 +506,11 @@ def _build_enhanced_notification( logger.info(f"DEBUG: Final status emoji: {status_emoji}") - base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {status_emoji}" + # Add total duration to base status + total_duration = _read_total_duration(artifact_dir) + duration_suffix = f" `{total_duration}`" if total_duration else "" + + base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {duration_suffix} {status_emoji}" notification_parts = [base_status] # Add job abort message right below overall status if applicable @@ -747,12 +751,16 @@ def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | No log_counts = _count_log_messages(step_dir) log_summary = _format_log_summary(log_counts) + # Read step duration + duration_str = _read_step_duration(step_dir) + duration_suffix = f" `{duration_str}`" if duration_str else "" + # Create step title - linked if MLflow URL available, plain-text otherwise if mlflow_run_url: mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_name) - step_title = f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}){log_summary}" + step_title = f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}){duration_suffix}{log_summary}" else: - step_title = f"#### {exit_status_emoji} {step_name}{log_summary}" + step_title = f"#### {exit_status_emoji} {step_name}{duration_suffix}{log_summary}" step_status.append(step_title) @@ -1393,6 +1401,55 @@ def _read_step_duration(step_dir: Path) -> str: return "" +def _read_total_duration(artifact_dir: Path | None) -> str: + """Read and sum test durations from all step directories.""" + + # Return empty duration if artifact_dir is None or unavailable + if artifact_dir is None or not artifact_dir.exists(): + return "" + + total_seconds = 0 + step_count = 0 + + # Scan all step directories for timing files + for step_dir in artifact_dir.iterdir(): + if not step_dir.is_dir(): + continue + + timing_file = step_dir / CI_METADATA_DIRNAME / "test_duration.yaml" + if not timing_file.exists(): + continue + + try: + with open(timing_file, encoding="utf-8") as f: + timing_data = yaml.safe_load(f) + + # Get raw duration in seconds for summing + duration_seconds = timing_data.get("duration", {}).get("seconds") + if duration_seconds is not None: + total_seconds += duration_seconds + step_count += 1 + + except Exception as timing_error: + logger.warning(f"Failed to read timing file {timing_file}: {timing_error}") + continue + + if step_count == 0: + return "" + + # Format total duration (similar to how individual step durations are formatted) + hours = int(total_seconds // 3600) + minutes = int((total_seconds % 3600) // 60) + seconds = int(total_seconds % 60) + + if hours > 0: + return f"{hours}h {minutes}m {seconds}s" + elif minutes > 0: + return f"{minutes}m {seconds}s" + else: + return f"{seconds}s" + + def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: """Process step logs from parent directory.""" From a5e3662810d59ebd25e8e0d77c3e60feb29e166e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 09:41:45 +0200 Subject: [PATCH 0857/1072] [caliper] engine: file_export: censoring_rules: make the Bearer rule more focused --- projects/caliper/engine/file_export/censoring_rules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index 1c3d695d9..df36d30a1 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -26,7 +26,8 @@ r"api[_-]?token\s*[:=]\s*\S+", r"refresh[_-]?token\s*[:=]\s*\S+", # Bearer tokens (compiled with IGNORECASE, so one pattern covers both cases) - r"Bearer\s+[A-Za-z0-9+/=]+", + # Require minimum 20 characters and avoid matching descriptive text like "bearer token file" + r"Bearer\s+[A-Za-z0-9+/=]{20,}", # Specific service API keys r"sk-[a-zA-Z0-9]{32,}", # OpenAI API keys r"ghp_[a-zA-Z0-9]{36}", # GitHub personal access tokens From dbf42a9ea9e841000c95b774b3f1653d34d174bf Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 10:50:37 +0200 Subject: [PATCH 0858/1072] [core] library: export_notifications: rework the relative links --- .../caliper/orchestration/notification.py | 64 +--- .../tests/test_notification_integration.py | 40 +- projects/core/library/export.py | 31 +- projects/core/library/export_notifications.py | 344 +++++++++--------- 4 files changed, 259 insertions(+), 220 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index baae62a49..68b81958f 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -61,12 +61,9 @@ def format_postprocess_status_notification( # Create step name as link to log file if available log_file = step_data.get("log_file") if log_file and get_file_link: - try: - log_url = get_file_link(log_file) - step_name_display = f"[**{step_name}**]({log_url})" - except Exception: - # Fallback to plain text if link generation fails - step_name_display = f"**{step_name}**" + step_name_display = get_file_link( + Path(status.base_directory) / log_file, text=f"**{step_name}**" + ) else: step_name_display = f"**{step_name}**" @@ -118,8 +115,8 @@ def _create_file_link(file_path: str, emoji: str, get_file_link: callable | None output_path = Path(file_path) filename = output_path.name - file_url = get_file_link(file_path) - return f" - {emoji} [{filename}]({file_url})" + file_link = get_file_link(file_path) + return f" - {emoji} {file_link}" except Exception: filename = file_path.split("/")[-1] return f" - {emoji} {filename}" @@ -160,32 +157,16 @@ def _format_artifacts_to_ai_data_step(step_data: dict, get_file_link: callable | # AI data directory link ai_data_dir = step_data.get("ai_data_dir") if ai_data_dir: - try: - # Extract relative path from the full path - ai_data_dir_relative = ai_data_dir.split("/")[-1] # Get just "ai_eval" - dir_url = get_file_link(ai_data_dir_relative) - lines.append(f" - 📁 [AI Eval Directory]({dir_url})") - except Exception: - lines.append(f" - 📁 AI Eval Directory: {ai_data_dir}") + # Extract relative path from the full path + ai_data_dir_relative = ai_data_dir.split("/")[-1] # Get just "ai_eval" + dir_link = get_file_link(ai_data_dir_relative, text="AI Eval Directory") + lines.append(f" - 📁 {dir_link}") # Output file link output_file = step_data.get("output_file") if output_file: - try: - import os - - output_file_relative = os.path.relpath( - output_file, - ai_data_dir or "", - ) - if ai_data_dir and "ai_eval" in ai_data_dir: - output_file_relative = f"ai_eval/{output_file_relative}" - file_url = get_file_link(output_file_relative) - filename = output_file.split("/")[-1] - lines.append(f" - 📄 [{filename}]({file_url})") - except Exception: - filename = output_file.split("/")[-1] - lines.append(f" - 📄 {filename}") + file_link = get_file_link(output_file) + lines.append(f" - 📄 {file_link}") return lines @@ -304,25 +285,10 @@ def _format_step_file_links( # Flatten the structure - just list all files without grouping by type for file_type, files in file_groups.items(): for file_path in files: - try: - # Combine output_dir with file_path if available - if output_dir: - # Use pathlib to properly join paths and avoid double slashes - from pathlib import Path - - full_path = str(Path(output_dir) / file_path) - else: - full_path = file_path - - file_url = get_file_link(full_path) - file_name = _get_display_name(file_path) - emoji = "📊" if file_type == "visualization" else "📄" - lines.append(f" - {emoji} [{file_name}]({file_url})") - except Exception: - # Fallback to plain text if link generation fails - file_name = _get_display_name(file_path) - emoji = "📊" if file_type == "visualization" else "📄" - lines.append(f" - {emoji} {file_name}") + full_path = Path(output_dir) / file_path + file_link = get_file_link(full_path, text=_get_display_name(file_path)) + emoji = "📊" if file_type == "visualization" else "📄" + lines.append(f" - {emoji} {file_link}") return lines diff --git a/projects/caliper/tests/test_notification_integration.py b/projects/caliper/tests/test_notification_integration.py index e60c22cc3..00ad8de1e 100644 --- a/projects/caliper/tests/test_notification_integration.py +++ b/projects/caliper/tests/test_notification_integration.py @@ -57,8 +57,24 @@ def test_notification_formatting_success(): ) # Mock file link function - def mock_get_file_link(file_path: str) -> str: - return f"https://example.com/artifacts/{file_path}" + def mock_get_file_link(file_path, text: str = None): + # Handle both Path objects and strings + if hasattr(file_path, "name"): + filename = file_path.name + path_str = str(file_path) + else: + path_str = str(file_path) + filename = path_str.split("/")[-1] + + # Strip base directory from path to get relative path + if "/test/artifacts/" in path_str: + relative_path = path_str.split("/test/artifacts/")[-1] + else: + # For paths that don't contain the base directory, keep the full relative path + relative_path = path_str + + display_text = text if text else filename + return f"[{display_text}](https://example.com/artifacts/{relative_path})" notification_text = format_postprocess_status_notification(status, mock_get_file_link) print(f"Generated notification text:\n{notification_text}\n") @@ -110,8 +126,24 @@ def test_notification_formatting_regression(): ], ) - def mock_get_file_link(file_path: str) -> str: - return f"https://example.com/artifacts/{file_path}" + def mock_get_file_link(file_path, text: str = None): + # Handle both Path objects and strings + if hasattr(file_path, "name"): + filename = file_path.name + path_str = str(file_path) + else: + path_str = str(file_path) + filename = path_str.split("/")[-1] + + # Strip base directory from path to get relative path + if "/test/artifacts/" in path_str: + relative_path = path_str.split("/test/artifacts/")[-1] + else: + # For paths that don't contain the base directory, keep the full relative path + relative_path = path_str + + display_text = text if text else filename + return f"[{display_text}](https://example.com/artifacts/{relative_path})" notification_text = format_postprocess_status_notification(status, mock_get_file_link) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 80eaa7223..891128e41 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -27,9 +27,10 @@ BackendResult, CaliperArtifactsExport, ExportStatus, + JobShutdown, TestPhase, _check_job_shutdown_status, - _create_mlflow_file_url_for_step, + _create_mlflow_url, send_notification, ) @@ -173,7 +174,10 @@ def _process_caliper_postprocess_status( step_subdir = step_dir.name def get_file_link(file_path: str, step_subdir=step_subdir) -> str: - return _create_mlflow_file_url_for_step(mlflow_run_url, step_subdir, file_path) + # Combine step_subdir and file_path into a single relative path + relative_path = Path(step_subdir) / file_path + result = _create_mlflow_url(mlflow_run_url, relative_path) + return result if result is not None else f"{mlflow_run_url}/{relative_path}" # Generate notification text from the structured result notification_text = format_postprocess_status_notification(result, get_file_link) @@ -327,7 +331,7 @@ def caliper_export_entrypoint( success=True, run_id="dry-run-mock-id", experiment_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123", - run_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", + run_url="http://DRY_RUN_MLFLOW_FAKE_URL/?workspace=forge-dry-run#/experiments/123/runs/dry-run-mock-id/artifacts", tracking_uri="http://DRY_RUN_MLFLOW_FAKE_URL", ) }, @@ -337,12 +341,25 @@ def caliper_export_entrypoint( message="Test execution completed with failures", ), ) + + # Check for job shutdown/abort status and add to mock status + shutdown_status = _check_job_shutdown_status() + if shutdown_status: + status.job_shutdown = JobShutdown.from_dict(shutdown_status) + logger.info(f"Added job shutdown status to dry run mock status: {shutdown_status}") else: status = run_caliper_orchestration_export( artifact_dir=artifact_dir, disable_censoring=disable_censoring, disable_file_export=disable_file_export, ) + + # Check for job shutdown/abort status and add to main export status + shutdown_status = _check_job_shutdown_status() + if shutdown_status: + status.job_shutdown = JobShutdown.from_dict(shutdown_status) + logger.info(f"Added job shutdown status to main export status: {shutdown_status}") + logger.info("Export status:\n" + yaml.dump(status.to_dict(), indent=4)) # Update fjob status with export results (only if file export is not disabled) @@ -356,11 +373,19 @@ def caliper_export_entrypoint( export_failed = True # Create failure status for notification status = ExportStatus(success=False, final_status=f"failed: {e}") + # Check for job shutdown/abort status and add to failure status + shutdown_status = _check_job_shutdown_status() + if shutdown_status: + status.job_shutdown = JobShutdown.from_dict(shutdown_status) except Exception as e: logger.exception(f"Export failed with unexpected error: {e}") export_failed = True # Create failure status for notification status = ExportStatus(success=False, final_status=f"failed: {e}") + # Check for job shutdown/abort status and add to failure status + shutdown_status = _check_job_shutdown_status() + if shutdown_status: + status.job_shutdown = JobShutdown.from_dict(shutdown_status) finally: # Send completion notifications regardless of success/failure diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index ece0f852c..8d9e9a01a 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -463,10 +463,13 @@ def _extract_finish_reason_from_status(status: ExportStatus) -> str: logger.info(f"DEBUG: Export status object: {status}") logger.info( - f"DEBUG: Status analysis - success={status.success}, censoring_occurred={status.censoring_occurred}" + f"DEBUG: Status analysis - success={status.success}, censoring_occurred={status.censoring_occurred}, job_shutdown={status.job_shutdown}" ) - if not status.success: + if status.job_shutdown and status.job_shutdown.is_aborted: + logger.info("DEBUG: Finish reason: aborted") + return "aborted" + elif not status.success: logger.info(f"DEBUG: Finish reason: failed (success={status.success})") return "failed" elif status.censoring_occurred: @@ -493,16 +496,19 @@ def _build_enhanced_notification( f"Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" ) - status_emoji = "✅" if success else "❌" - logger.info(f"Initial emoji based on success: {status_emoji}") - - if censoring_occurred: - status_emoji = "⚠️" - logger.info(f"Changed emoji to warning due to censoring: {status_emoji}") - - if finish_reason == "failed": + # Determine status emoji with abort taking precedence + if status.job_shutdown and status.job_shutdown.is_aborted: + status_emoji = "🛑" + logger.info(f"Setting emoji to stop sign due to job abort: {status_emoji}") + elif not success: status_emoji = "❌" - logger.info(f"Changed emoji to failed due to finish_reason: {status_emoji}") + logger.info(f"Setting emoji to failed due to success={success}: {status_emoji}") + elif censoring_occurred: + status_emoji = "⚠️" + logger.info(f"Setting emoji to warning due to censoring: {status_emoji}") + else: + status_emoji = "✅" + logger.info(f"Setting emoji to success: {status_emoji}") logger.info(f"DEBUG: Final status emoji: {status_emoji}") @@ -510,7 +516,10 @@ def _build_enhanced_notification( total_duration = _read_total_duration(artifact_dir) duration_suffix = f" `{total_duration}`" if total_duration else "" - base_status = f"{status_emoji} **Execution of `{fjob_project}` {fjob_args_str}** {duration_suffix} {status_emoji}" + if fjob_args_str: + fjob_args_str = f" {fjob_args_str}" + + base_status = f"{status_emoji} **Execution of `{fjob_project}`{fjob_args_str}** {duration_suffix} {status_emoji}" notification_parts = [base_status] # Add job abort message right below overall status if applicable @@ -756,11 +765,9 @@ def _get_step_status_section(artifact_dir: Path | None, mlflow_run_url: str | No duration_suffix = f" `{duration_str}`" if duration_str else "" # Create step title - linked if MLflow URL available, plain-text otherwise - if mlflow_run_url: - mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_name) - step_title = f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}){duration_suffix}{log_summary}" - else: - step_title = f"#### {exit_status_emoji} {step_name}{duration_suffix}{log_summary}" + + mlflow_log_link = _create_mlflow_file_link(step_name, mlflow_run_url, step_dir / "run.log") + step_title = f"#### {exit_status_emoji} {mlflow_log_link}{duration_suffix}{log_summary}" step_status.append(step_title) @@ -892,16 +899,8 @@ def _get_postprocess_status_links( f"Postprocess status - {postprocess_status_file} parsed result successfully" ) - # Create file link function for this step - def get_file_link(file_path: Path, step_subdir: str = step_name) -> str: - if mlflow_run_url: - # Create MLflow artifact URL - return _create_mlflow_file_url_for_step( - mlflow_run_url, step_subdir, str(file_path) - ) - else: - # Fallback: just return the file path as text - return str(file_path) + # Use shared unified get_file_link function + get_file_link = _create_unified_get_file_link(mlflow_run_url) # Generate notification text from the structured result logger.info( @@ -955,46 +954,142 @@ def _extract_artifact_links(status: ExportStatus) -> tuple[list[str], str | None return artifact_links, mlflow_run_url -def _create_mlflow_file_url_for_step( - mlflow_run_url: str, step_dir_name: str, file_path: str -) -> str: - """Create MLflow URL for a specific file within a step directory. +def _create_unified_get_file_link(mlflow_run_url: str | None) -> callable: + """Create a unified get_file_link function that always expects absolute paths. Args: + mlflow_run_url: Base MLflow run URL, or None for fallback mode + + Returns: + get_file_link function that takes absolute Path and returns URL + """ + + def get_file_link(file_path: Path, text: str = None) -> str: + if not file_path: + return "NO_FILE_RECEIVED" + + return _create_mlflow_file_link(text or file_path.name, mlflow_run_url, file_path) + + return get_file_link + + +def _create_mlflow_file_link(text: str, mlflow_run_url: str, file_path: Path) -> str: + """Create MLflow link for a specific file. + + Args: + text: Text to display in the link mlflow_run_url: Base MLflow run URL - step_dir_name: Name of the step directory - file_path: Relative path to file from step directory + file_path: Absolute path to file Returns: - Full MLflow URL to the file + Markdown link format `[text](url)` if successful, `text (**error description**)` if not + """ + if not mlflow_run_url: + return text - Raises: - ValueError: If URL format is unexpected + # Check file existence - still create link but with warning if file doesn't exist locally + file_exists_locally = file_path.exists() + if not file_exists_locally: + logger.warning(f"File not found locally but creating MLflow link anyway: {file_path}") + # We'll still create the link since the file might exist in MLflow + + if not file_path.is_absolute(): + return f"{text} (**path not absolute**)" + + try: + # Convert absolute path to relative path for MLflow URL + artifacts_base_dir = env.BASE_ARTIFACT_DIR + try: + relative_path = file_path.relative_to(artifacts_base_dir) + except ValueError: + # File not under artifacts base, use the full path + relative_path = file_path + + # Create the MLflow URL + url = _create_mlflow_url(mlflow_run_url, relative_path) + if url is None: + return f"{text} (**URL creation failed**)" + + # Add indicator if file doesn't exist locally but we're still linking to MLflow + if not file_exists_locally: + return f"[{text}]({url}) ⚠️" + else: + return f"[{text}]({url})" + except Exception as e: + logger.error(f"Failed to create MLflow link for {file_path}: {e}") + return f"{text} (**link creation failed**)" + + +def _create_mlflow_url(mlflow_run_url: str, file_path: Path) -> str | None: + """Create MLflow URL for artifacts. + + Args: + mlflow_run_url: Base MLflow run URL + file_path: Relative path within artifacts (e.g., Path("01__test/run.log") or Path("data.csv")) + + Returns: + Full MLflow URL to the artifact, or None/fallback URL if format is unexpected """ - if "/artifacts" not in mlflow_run_url: - raise ValueError(f"Unexpected MLflow URL format: {mlflow_run_url}") - - # Clean file path - file_clean = file_path.lstrip("/") - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" - else: - base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_url}/artifacts/{step_dir_name}/{file_clean}{workspace_param}" + from urllib.parse import urlparse + + if not mlflow_run_url: + return f"BASE_URL_MISSING/{file_path}" + + try: + if "/artifacts" not in mlflow_run_url: + logger.error(f"Unexpected MLflow URL format: {mlflow_run_url}") + return None + + # Convert file_path to string with forward slashes + file_path_str = str(file_path).replace("\\", "/") + + # Parse the URL properly + from urllib.parse import parse_qs + + parsed = urlparse(mlflow_run_url) + + # Handle case where there's no hash fragment - normalize to hash-based format + if not parsed.fragment: + raise ValueError(f"Hash fragment missing in the MLFLow URL {mlflow_run_url} ...") + + # Parse the fragment to extract any query parameters within it + fragment = parsed.fragment + fragment_path = fragment + workspace_param = None + + # Check if there are query parameters in the fragment (like ?workspace=forge-sandbox) + if "?" in fragment: + fragment_path, fragment_query = fragment.split("?", 1) + fragment_params = parse_qs(fragment_query) + + # Extract workspace parameter if present + if "workspace" in fragment_params: + workspace_param = fragment_params["workspace"][0] + + # Build base URL + base_url = f"{parsed.scheme}://{parsed.netloc}" + + # Add path if present + if parsed.path and parsed.path != "/": + base_url += parsed.path + + # Combine query parameters: main URL query + workspace from fragment + query_parts = [] + if parsed.query: + query_parts.append(parsed.query) + if workspace_param: + query_parts.append(f"workspace={workspace_param}") + + if query_parts: + base_url += f"?{'&'.join(query_parts)}" + + # Clean fragment path (remove trailing slash) + fragment_path = fragment_path.rstrip("/") + return f"{base_url}#{fragment_path}/{file_path_str}" + + except Exception as e: + logger.error(f"Failed to construct MLflow URL for {file_path}: {e}") + return None def _process_notification_files(step_dir: Path) -> list[str]: @@ -1021,8 +1116,8 @@ def _process_notification_files(step_dir: Path) -> list[str]: return notifications_from_files -def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: - """Extract test execution information from __test_labels__.yaml files. +def _extract_test_metadata_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: + """Extract test execution information from the test metadata files. Args: artifact_dir: Directory to search for __test_labels__.yaml files @@ -1033,23 +1128,23 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N """ test_info_lines = [] - # Search for __test_labels__.yaml files recursively - test_labels_files = list(artifact_dir.glob("**/__test_labels__.yaml")) + # Search for the test medata files recursively + test_metadata_files = _search_caliper_metadata_files(artifact_dir) - if not test_labels_files: + if not test_metadata_files: return [] - for test_labels_file in test_labels_files: + for test_metadata_file in test_metadata_files: try: - with open(test_labels_file, encoding="utf-8") as f: - test_data = yaml.safe_load(f) or {} + with open(test_metadata_file, encoding="utf-8") as f: + test_metadata = yaml.safe_load(f) or {} # Extract directory relative to artifact_dir - use just the immediate directory name - relative_dir = test_labels_file.parent.relative_to(artifact_dir) + relative_dir = test_metadata_file.parent.relative_to(artifact_dir) dir_name = relative_dir.name if relative_dir != Path(".") else "root" # Extract completion info - completion = test_data.get("completion", {}) + completion = test_metadata.get("completion", {}) success = completion.get("success") message = completion.get("message") @@ -1067,24 +1162,14 @@ def _extract_test_labels_info(artifact_dir: Path, mlflow_run_url: str | None = N status_emoji = "❓" # Create link to __test_labels__.yaml file if MLflow URL is available - if mlflow_run_url: - try: - # Get the step directory name (relative to parent) - step_dir_name = str(relative_dir) - test_labels_url = _create_mlflow_file_url_for_step( - mlflow_run_url, step_dir_name, "__test_labels__.yaml" - ) - dir_link = f"[**{dir_name}**]({test_labels_url})" - except Exception as e: - logger.warning(f"Failed to create MLflow link for {test_labels_file}: {e}") - dir_link = f"**{dir_name}**" - else: - dir_link = f"**{dir_name}**" + dir_link = _create_mlflow_file_link( + f"**{dir_name}**", mlflow_run_url, test_metadata_file + ) test_info_lines.append(f"* {status_emoji} {dir_link}{message}") except Exception as e: - test_info_lines.append(f"**{test_labels_file.name}**: Error reading file - {e}") + test_info_lines.append(f"**{test_metadata_file.name}**: Error reading file - {e}") return test_info_lines @@ -1150,29 +1235,14 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> # Extract test labels for this specific step try: - test_labels_info = _extract_test_labels_info(step_dir, mlflow_run_url) - if test_labels_info: - step_details.extend(test_labels_info) + test_metadata_info = _extract_test_metadata_info(step_dir, mlflow_run_url) + if test_metadata_info: + step_details.extend(test_metadata_info) except Exception as e: logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") - # Create file link function for this step (shared by multiple extractors) - def get_file_link(file_path: Path) -> str: - if mlflow_run_url: - # Create MLflow artifact URL - convert absolute path to relative to step directory - try: - relative_file_path = file_path.relative_to(step_dir) - return _create_mlflow_file_url_for_step( - mlflow_run_url, step_dir.name, str(relative_file_path) - ) - except ValueError: - # If file_path is not under step_dir, use the file name only - return _create_mlflow_file_url_for_step( - mlflow_run_url, step_dir.name, file_path.name - ) - else: - # Fallback: just return the file path as text - return str(file_path) + # Use shared unified get_file_link function + get_file_link = _create_unified_get_file_link(mlflow_run_url) # Extract caliper metadata for this specific step (before postprocess status) try: @@ -1333,56 +1403,6 @@ def _get_overall_status_from_steps(artifact_dir: Path) -> str: return "🔴" # Error checking = red -def _create_mlflow_url(mlflow_run_url: str, step_dir_name: str) -> str | None: - """Create MLflow URL for step logs.""" - - if not mlflow_run_url: - return f"BASE_URL_MISSING/{step_dir_name}" - - if "/artifacts" not in mlflow_run_url: - logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") - return None - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, params = hash_fragment.split("/artifacts", 1) - workspace_param = params if "?workspace=" in params else "" - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}/run.log{workspace_param}" - else: - base_url, params = mlflow_run_url.split("/artifacts", 1) - workspace_param = params if "?workspace=" in params else "" - return f"{base_url}/artifacts/{step_dir_name}/run.log{workspace_param}" - - -def _create_mlflow_step_url(mlflow_run_url: str, step_dir_name: str) -> str | None: - """Create MLflow URL for step directory (for file access).""" - if "/artifacts" not in mlflow_run_url: - logger.warning(f"Unexpected MLflow URL format: {mlflow_run_url}") - return None - - if "#" in mlflow_run_url: - base_domain, hash_fragment = mlflow_run_url.split("#", 1) - if "/artifacts" not in hash_fragment: - raise ValueError("Artifacts not found in hash fragment") - - hash_base, artifacts_part = hash_fragment.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_domain}#{hash_base}/artifacts/{step_dir_name}{workspace_param}" - else: - base_url, artifacts_part = mlflow_run_url.split("/artifacts", 1) - # Extract workspace parameter if present, ignoring existing path - workspace_param = "" - if "?workspace=" in artifacts_part: - workspace_param = artifacts_part[artifacts_part.find("?") :] - return f"{base_url}/artifacts/{step_dir_name}{workspace_param}" - - def _read_step_duration(step_dir: Path) -> str: """Read step duration from timing file.""" @@ -1470,33 +1490,29 @@ def _process_step_status(artifact_dir: Path, mlflow_run_url: str) -> list[str]: if not run_log.exists(): continue - mlflow_log_url = _create_mlflow_url(mlflow_run_url, step_dir.name) - if not mlflow_log_url: - mlflow_log_url = "NO_URL" - step_name = step_dir.name.replace("__", " ").replace("_", " ").title() + mlflow_log_link = _create_mlflow_url(step_name, mlflow_run_url, run_log) + duration_str = _read_step_duration(step_dir) exit_status_emoji, exit_status = _read_step_exit_status(step_dir, current_step_name) step_status.append("") if duration_str: - step_status.append( - f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url}) `{duration_str}`" - ) + step_status.append(f"#### {exit_status_emoji} {mlflow_log_link} `{duration_str}`") else: - step_status.append(f"#### {exit_status_emoji} [{step_name}]({mlflow_log_url})") + step_status.append(f"#### {exit_status_emoji} {mlflow_log_link}") step_status.extend(_process_notification_files(step_dir)) step_details = _process_step_details(step_dir, mlflow_run_url) # Add test execution info for this step - if step_details["test_labels_info"]: + if step_details["test_metadata_info"]: # Add Test Execution Overview if we have any step info step_status.append("") step_status.append("**Test Execution Overview**") - step_status.extend([f"* {info}" for info in step_details["test_labels_info"]]) + step_status.extend([f"* {info}" for info in step_details["test_metadata_info"]]) # Add postprocess info for this step if step_details["postprocess_info"]: From bca687586114f76fd08d7edb140c6d14fdc9f35c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 19:49:20 +0200 Subject: [PATCH 0859/1072] [caliper] Fix the notification links --- projects/caliper/orchestration/export.py | 2 +- .../caliper/orchestration/notification.py | 27 +++++++------ projects/core/library/export.py | 21 ++++++++-- projects/core/library/export_notifications.py | 39 ++++++------------- 4 files changed, 46 insertions(+), 43 deletions(-) diff --git a/projects/caliper/orchestration/export.py b/projects/caliper/orchestration/export.py index 87bffc8d7..c7c3b8e94 100644 --- a/projects/caliper/orchestration/export.py +++ b/projects/caliper/orchestration/export.py @@ -522,7 +522,7 @@ def build_mlflow_run_url( assert_tracking_uri_has_no_userinfo(tracking_uri) qs = f"?workspace={quote(workspace, safe='')}" if workspace else "" - return f"{tracking_uri}/{qs}#/experiments/{experiment_id}/runs/{run_id}/artifacts" + return f"{tracking_uri}#/experiments/{experiment_id}/runs/{run_id}/artifacts{qs}" def _discover_precreated_mlflow_run_id(from_path: Path) -> str | None: diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 68b81958f..6b03faa34 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -21,7 +21,7 @@ def format_postprocess_status_notification( Args: status: Typed PostprocessStatus object get_file_link: Optional callback function that takes a file path and returns a URL. - Signature: get_file_link(file_path: str) -> str + Signature: get_file_link(file_path: Path | str) -> str Returns: Formatted notification text to include in GitHub notification @@ -35,8 +35,12 @@ def format_postprocess_status_notification( # Check overall status (keep unchanged regardless of abort status) status_emoji = "✅" if status.is_success() else "❌" base_directory = Path(status.base_directory) - if base_directory.name == "status_files": - base_directory = base_directory.parent + + def postprocess_get_file_link(file_path: Path | str, text: str = None) -> str: + if not get_file_link: + return text + + return get_file_link(base_directory / file_path, text) lines.append(f"**Post-processing Status** {status_emoji} `{base_directory}`") @@ -60,10 +64,8 @@ def format_postprocess_status_notification( # Create step name as link to log file if available log_file = step_data.get("log_file") - if log_file and get_file_link: - step_name_display = get_file_link( - Path(status.base_directory) / log_file, text=f"**{step_name}**" - ) + if log_file: + step_name_display = postprocess_get_file_link(log_file, text=f"**{step_name}**") else: step_name_display = f"**{step_name}**" @@ -78,7 +80,9 @@ def format_postprocess_status_notification( lines.append(f" * `{reason}`") # Use object-oriented step formatter to handle step-specific details - step_details = _format_step_details_with_formatters(step_name, step_data, get_file_link) + step_details = _format_step_details_with_formatters( + step_name, step_data, postprocess_get_file_link + ) lines.extend(step_details) return "\n".join(lines) if lines else "" @@ -115,7 +119,8 @@ def _create_file_link(file_path: str, emoji: str, get_file_link: callable | None output_path = Path(file_path) filename = output_path.name - file_link = get_file_link(file_path) + # Pass Path object to get_file_link + file_link = get_file_link(output_path) return f" - {emoji} {file_link}" except Exception: filename = file_path.split("/")[-1] @@ -159,13 +164,13 @@ def _format_artifacts_to_ai_data_step(step_data: dict, get_file_link: callable | if ai_data_dir: # Extract relative path from the full path ai_data_dir_relative = ai_data_dir.split("/")[-1] # Get just "ai_eval" - dir_link = get_file_link(ai_data_dir_relative, text="AI Eval Directory") + dir_link = get_file_link(Path(ai_data_dir_relative), text="AI Eval Directory") lines.append(f" - 📁 {dir_link}") # Output file link output_file = step_data.get("output_file") if output_file: - file_link = get_file_link(output_file) + file_link = get_file_link(Path(output_file)) lines.append(f" - 📄 {file_link}") return lines diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 891128e41..60d153aaa 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -36,6 +36,10 @@ logger = logging.getLogger(__name__) +# Global switch to disable test failure when censoring occurs +# Set to True to disable test failures when secrets are found and censored +DISABLE_CENSORING_TEST_FAILURE = False + class FinishReason(StrEnum): SUCCESS = "success" @@ -173,7 +177,11 @@ def _process_caliper_postprocess_status( # Fallback to step_dir.name for backward compatibility step_subdir = step_dir.name - def get_file_link(file_path: str, step_subdir=step_subdir) -> str: + def get_file_link(file_path: str | Path, step_subdir=step_subdir) -> str: + # Convert to string if Path object was passed + if isinstance(file_path, Path): + file_path = str(file_path) + # Combine step_subdir and file_path into a single relative path relative_path = Path(step_subdir) / file_path result = _create_mlflow_url(mlflow_run_url, relative_path) @@ -331,7 +339,7 @@ def caliper_export_entrypoint( success=True, run_id="dry-run-mock-id", experiment_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123", - run_url="http://DRY_RUN_MLFLOW_FAKE_URL/?workspace=forge-dry-run#/experiments/123/runs/dry-run-mock-id/artifacts", + run_url="http://DRY_RUN_MLFLOW_FAKE_URL/#/experiments/123/runs/dry-run-mock-id/artifacts?workspace=forge-dry-run", tracking_uri="http://DRY_RUN_MLFLOW_FAKE_URL", ) }, @@ -417,9 +425,14 @@ def caliper_export_entrypoint( if export_failed or notification_failed: return 1, "failed" - # Check if censoring occurred and return exit code 1 if so + # Check if censoring occurred and return exit code 1 if so (unless disabled) if status and status.censoring_occurred: - return 1, "censoring_occurred" + if DISABLE_CENSORING_TEST_FAILURE: + logger.info( + "Censoring occurred but test failure disabled via DISABLE_CENSORING_TEST_FAILURE" + ) + else: + return 1, "censoring_occurred" return 0 diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 8d9e9a01a..e5b2fa57d 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -460,23 +460,13 @@ def _get_project_and_args(project: str) -> tuple[str, str]: def _extract_finish_reason_from_status(status: ExportStatus) -> str: """Extract finish reason from status.""" - logger.info(f"DEBUG: Export status object: {status}") - - logger.info( - f"DEBUG: Status analysis - success={status.success}, censoring_occurred={status.censoring_occurred}, job_shutdown={status.job_shutdown}" - ) - if status.job_shutdown and status.job_shutdown.is_aborted: - logger.info("DEBUG: Finish reason: aborted") return "aborted" elif not status.success: - logger.info(f"DEBUG: Finish reason: failed (success={status.success})") return "failed" elif status.censoring_occurred: - logger.info("DEBUG: Finish reason: completed with censoring") return "completed with censoring" else: - logger.info("DEBUG: Finish reason: completed") return "completed" @@ -499,18 +489,12 @@ def _build_enhanced_notification( # Determine status emoji with abort taking precedence if status.job_shutdown and status.job_shutdown.is_aborted: status_emoji = "🛑" - logger.info(f"Setting emoji to stop sign due to job abort: {status_emoji}") elif not success: status_emoji = "❌" - logger.info(f"Setting emoji to failed due to success={success}: {status_emoji}") elif censoring_occurred: status_emoji = "⚠️" - logger.info(f"Setting emoji to warning due to censoring: {status_emoji}") else: status_emoji = "✅" - logger.info(f"Setting emoji to success: {status_emoji}") - - logger.info(f"DEBUG: Final status emoji: {status_emoji}") # Add total duration to base status total_duration = _read_total_duration(artifact_dir) @@ -837,19 +821,13 @@ def _get_postprocess_status_links( artifact_dir: Path | None, mlflow_run_url: str | None ) -> list[str]: """Get postprocess status links.""" - logger.info( - f"DEBUG: Postprocess status - artifact_dir={artifact_dir}, mlflow_run_url={mlflow_run_url}" - ) - if not artifact_dir or not artifact_dir.exists(): - logger.info("DEBUG: Postprocess status - artifact_dir missing or doesn't exist") return [] step_log_links = [] # Look for postprocess results in step directories step_dirs = sorted(artifact_dir.glob("*")) - logger.info(f"DEBUG: Postprocess status - found {len(step_dirs)} directories in {artifact_dir}") for step_dir in step_dirs: if not step_dir.is_dir() or step_dir.name.startswith("."): @@ -964,10 +942,15 @@ def _create_unified_get_file_link(mlflow_run_url: str | None) -> callable: get_file_link function that takes absolute Path and returns URL """ - def get_file_link(file_path: Path, text: str = None) -> str: + def get_file_link(file_path: Path | str, text: str = None) -> str: if not file_path: return "NO_FILE_RECEIVED" + # Convert to Path object if it's a string, but warn that Path objects are preferred + if isinstance(file_path, str): + logger.warning(f"get_file_link received string instead of Path object: {file_path}") + file_path = Path(file_path) + return _create_mlflow_file_link(text or file_path.name, mlflow_run_url, file_path) return get_file_link @@ -998,7 +981,9 @@ def _create_mlflow_file_link(text: str, mlflow_run_url: str, file_path: Path) -> try: # Convert absolute path to relative path for MLflow URL - artifacts_base_dir = env.BASE_ARTIFACT_DIR + artifacts_base_dir = Path( + os.environ.get("BASE_ARTIFACT_DIR") or env.BASE_ARTIFACT_DIR + ).parent try: relative_path = file_path.relative_to(artifacts_base_dir) except ValueError: @@ -1640,9 +1625,9 @@ def _format_caliper_metadata_info_for_step( # Format path with link to metadata file if get_file_link: metadata_file_link = get_file_link(metadata_file) - path_info = f"* 📊 Test directory: [`{display_path}`]({metadata_file_link})" + path_info = f" * 📊 Test directory: [`{display_path}`]({metadata_file_link})" else: - path_info = f"* 📊 Test directory: `{display_path}`" + path_info = f" * 📊 Test directory: `{display_path}`" metadata_lines.append(path_info) # Look for completion information @@ -1681,7 +1666,7 @@ def _format_caliper_metadata_info_for_step( metadata_file.parent.relative_to(base_dir) if base_dir else metadata_file.parent ) metadata_lines.append( - f"* 📊 Test directory: `{relative_path}` - Error reading metadata: {e}" + f" * 📊 Test directory: `{relative_path}` - Error reading metadata: {e}" ) return metadata_lines From b5b77764eaa0aaa7154854d43d9732abb94bf2f8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Mon, 14 Sep 2026 22:19:56 +0200 Subject: [PATCH 0860/1072] [caliper] Reduce the false positive in the regex censoring --- .../engine/file_export/censoring_rules.py | 48 ++++++---- projects/caliper/tests/test_censoring.py | 89 +++++++++++++++++++ projects/core/library/export.py | 2 +- 3 files changed, 122 insertions(+), 17 deletions(-) diff --git a/projects/caliper/engine/file_export/censoring_rules.py b/projects/caliper/engine/file_export/censoring_rules.py index df36d30a1..f6a30bd3d 100644 --- a/projects/caliper/engine/file_export/censoring_rules.py +++ b/projects/caliper/engine/file_export/censoring_rules.py @@ -12,19 +12,19 @@ # Keyword patterns to detect in file content # These are compiled regex patterns that match common sensitive data patterns KEYWORD_PATTERNS = [ - # Password patterns - r"password\s*[:=]\s*\S+", - r"passwd\s*[:=]\s*\S+", - # API key patterns - r"api[_-]?key\s*[:=]\s*\S+", - r"apikey\s*[:=]\s*\S+", - r"api[_-]?secret\s*[:=]\s*\S+", - # Token patterns (exclude version specifiers like ==0.14.0) + # Password patterns - exclude function calls, env lookups, already redacted content + r"password\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[|''|\"\"|\$)['\"]?[^'\"\s<>\[\]()]{3,}", + r"passwd\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[|''|\"\"|\$)['\"]?[^'\"\s<>\[\]()]{3,}", + # API key patterns - improved to exclude function calls and already redacted content + r"api[_-]?key\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[|''|\"\"|\$)['\"]?[^'\"\s<>\[\]()]{3,}", + r"apikey\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[|''|\"\"|\$)['\"]?[^'\"\s<>\[\]()]{3,}", + r"api[_-]?secret\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[|''|\"\"|\$)['\"]?[^'\"\s<>\[\]()]{3,}", + # Token patterns (general token pattern still disabled, but specific ones enabled) # r"(?:^|[^a-zA-Z])token\s*[:=]\s*(?![=~<>])\S+", # DISABLED, too sensitive for inference work ... - r"secret[_-]?token\s*[:=]\s*\S+", - r"access[_-]?token\s*[:=]\s*\S+", - r"api[_-]?token\s*[:=]\s*\S+", - r"refresh[_-]?token\s*[:=]\s*\S+", + r"secret[_-]?token\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", + r"access[_-]?token\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", + r"api[_-]?token\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", + r"refresh[_-]?token\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", # Bearer tokens (compiled with IGNORECASE, so one pattern covers both cases) # Require minimum 20 characters and avoid matching descriptive text like "bearer token file" r"Bearer\s+[A-Za-z0-9+/=]{20,}", @@ -37,14 +37,30 @@ r"ghr_[a-zA-Z0-9]{36}", # GitHub refresh tokens # AWS patterns r"AKIA[0-9A-Z]{16}", # AWS Access Key ID - r"aws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*\S+", + r"aws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", # Database connection strings r"mongodb://[^/\s]+:[^@\s]+@", r"mysql://[^/\s]+:[^@\s]+@", r"postgresql://[^/\s]+:[^@\s]+@", - # Generic credential patterns - r"credential\s*[:=]\s*\S+", - r"private[_-]?key\s*[:=]\s*\S+", + # Generic credential patterns - improved + r"credential\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", + r"private[_-]?key\s*[:=]\s*(?!get_|os\.|env\.|<|>|\[)['\"]?[^'\"\s<>\[\]()]{3,}", + # ML/AI service tokens + r"(?:huggingface|anthropic|openai|claude)[_-]?(?:token|key|api[_-]?key)\s*[:=]\s*[^'\"\s<>\[\]()]{3,}", + r"hf_[a-zA-Z0-9]{20,}", # Hugging Face tokens + r"ant-[a-zA-Z0-9]{20,}", # Anthropic API keys (if they use this format) + # Common secret/key patterns + r"(?:jwt|webhook|slack|discord|telegram)[_-]?(?:secret|token|key)\s*[:=]\s*[^'\"\s<>\[\]()]{3,}", + r"encryption[_-]?key\s*[:=]\s*[^'\"\s<>\[\]()]{3,}", + r"signing[_-]?key\s*[:=]\s*[^'\"\s<>\[\]()]{3,}", + # Slack tokens (specific patterns) + r"xox[bpoa]-[a-zA-Z0-9-]+", # Slack tokens (bot, app, oauth, etc.) + # Command line arguments + r"--(?:password|api[_-]?key|token|secret)\s+[^'\"\s]{3,}", + # OpenShift/Kubernetes secret creation commands + r"(?:oc|kubectl)\s+create\s+secret\s+.*--from-literal[=\s]+[^=\s]+=[^'\"\s]{3,}", + # Environment variable exports in shell (explicit lowercase to avoid matching var names) + r"(?:^|\s)[e][x][p][o][r][t]\s+[A-Z_]*(?:PASSWORD|API_KEY|TOKEN|SECRET)[A-Z_]*=[^'\"\s]{3,}", ] # Compile patterns for better performance diff --git a/projects/caliper/tests/test_censoring.py b/projects/caliper/tests/test_censoring.py index 6fd9d7d60..34726e040 100644 --- a/projects/caliper/tests/test_censoring.py +++ b/projects/caliper/tests/test_censoring.py @@ -195,6 +195,95 @@ def test_no_duplicate_bearer_pattern(self): f"Expected exactly 1 bearer pattern, found {len(bearer_patterns)}: {bearer_patterns}" ) + def test_improved_patterns_avoid_false_positives(self): + """Test that improved patterns avoid common false positives.""" + # These should NOT be flagged as secrets + false_positives = [ + "password = get_password_from_user()", + "api_key = os.environ.get('API_KEY')", + "api_key = ''", + "password: ", + "api_key: [MASKED]", + "api_key=${API_KEY}", + ] + + for text in false_positives: + matches = self._matches_any_pattern(text) + assert not matches, f"False positive detected: '{text}'" + + def test_new_ml_ai_service_patterns(self): + """Test detection of ML/AI service tokens.""" + ml_patterns = [ + "huggingface_token=hf_abcd1234567890", + "anthropic_api_key=ant-abc123", + "openai_api_key=sk-proj123456789", + "claude_token=ant-123456", + "hf_abcdef1234567890123456", # Direct HF token format + ] + + for text in ml_patterns: + matches = self._matches_any_pattern(text) + assert matches, f"ML/AI pattern not detected: '{text}'" + + def test_command_line_patterns(self): + """Test detection of command-line secrets.""" + cli_patterns = [ + "--password secret123", + "--api-key mykey456", + "--token bearer_token", + "export API_KEY=secret123", + "export DATABASE_PASSWORD=prod456", + "oc create secret generic test --from-literal=password=secret123", + "kubectl create secret generic test --from-literal=user=admin", + ] + + for text in cli_patterns: + matches = self._matches_any_pattern(text) + assert matches, f"Command-line pattern not detected: '{text}'" + + def test_webhook_and_service_secrets(self): + """Test detection of webhook and service secrets.""" + service_patterns = [ + "jwt_secret=mysecret123", + "webhook_secret=github789", + "slack_token=xoxb-123456789", + "encryption_key=enc123456", + "signing_key=sign789", + "xoxp-1234567890-abcdef", # Slack app token format + ] + + for text in service_patterns: + matches = self._matches_any_pattern(text) + assert matches, f"Service pattern not detected: '{text}'" + + def test_export_pattern_avoids_variable_names(self): + """Test that export pattern only catches shell exports, not variable names.""" + # These should NOT be flagged (false positives fixed) + false_positives = [ + "PSAP_FORGE_AWS_S3_EXPORT_SECRET_PATH=/var/run/secrets/fournos/psap-forge-aws-s3-value", + "PSAP_FORGE_MLFLOW_EXPORT_SECRET_PATH=/var/run/secrets/fournos/psap-forge-mlflow-value", + "MY_EXPORT_SECRET_CONFIG=some_value", + "EXPORT_API_KEY_PATH=/path/to/file", + "CONFIG_EXPORT_PASSWORD_FILE=/etc/config", + ] + + for text in false_positives: + matches = self._matches_any_pattern(text) + assert not matches, f"False positive detected in export pattern: '{text}'" + + # These SHOULD be flagged (legitimate shell exports) + legitimate_exports = [ + "export API_KEY=secret123", + "export DATABASE_PASSWORD=prod456", + "export SECRET_TOKEN=mytoken789", + " export WEBHOOK_SECRET=hook123", # with leading whitespace + "\texport SLACK_TOKEN=slack456", # with tab + ] + + for text in legitimate_exports: + matches = self._matches_any_pattern(text) + assert matches, f"Legitimate export not detected: '{text}'" + # --------------------------------------------------------------------------- # CensoringResult diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 60d153aaa..05c5b843d 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -38,7 +38,7 @@ # Global switch to disable test failure when censoring occurs # Set to True to disable test failures when secrets are found and censored -DISABLE_CENSORING_TEST_FAILURE = False +DISABLE_CENSORING_TEST_FAILURE = True class FinishReason(StrEnum): From 3e9a44a496754cd81e0aa8519f62f3f8404b3290 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 09:22:11 +0200 Subject: [PATCH 0861/1072] [core] library: export_notifications: finish updating --- projects/core/library/export_notifications.py | 79 ++++++++----------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index e5b2fa57d..e861eac1d 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -878,7 +878,7 @@ def _get_postprocess_status_links( ) # Use shared unified get_file_link function - get_file_link = _create_unified_get_file_link(mlflow_run_url) + get_file_link = _create_get_file_link(mlflow_run_url) # Generate notification text from the structured result logger.info( @@ -932,7 +932,7 @@ def _extract_artifact_links(status: ExportStatus) -> tuple[list[str], str | None return artifact_links, mlflow_run_url -def _create_unified_get_file_link(mlflow_run_url: str | None) -> callable: +def _create_get_file_link(mlflow_run_url: str | None) -> callable: """Create a unified get_file_link function that always expects absolute paths. Args: @@ -1065,12 +1065,15 @@ def _create_mlflow_url(mlflow_run_url: str, file_path: Path) -> str | None: if workspace_param: query_parts.append(f"workspace={workspace_param}") - if query_parts: - base_url += f"?{'&'.join(query_parts)}" - # Clean fragment path (remove trailing slash) fragment_path = fragment_path.rstrip("/") - return f"{base_url}#{fragment_path}/{file_path_str}" + + url = f"{base_url}#{fragment_path}/{file_path_str}" + + if query_parts: + url += f"?{'&'.join(query_parts)}" + + return url except Exception as e: logger.error(f"Failed to construct MLflow URL for {file_path}: {e}") @@ -1180,6 +1183,8 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: # Extract directory relative to artifact_dir relative_dir = postprocess_file.parent.relative_to(artifact_dir) + if relative_dir.name == "status_files": + relative_dir = relative_dir.parent dir_name = str(relative_dir) if relative_dir != Path(".") else "root" # Extract overall status @@ -1226,33 +1231,26 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> except Exception as e: logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") - # Use shared unified get_file_link function - get_file_link = _create_unified_get_file_link(mlflow_run_url) + get_file_link = _create_get_file_link(mlflow_run_url) - # Extract caliper metadata for this specific step (before postprocess status) + # Extract caliper metadata for this specific step try: - metadata_files = _search_caliper_metadata_files(step_dir) - metadata_info = _format_caliper_metadata_info_for_step( - metadata_files, get_file_link, step_dir.parent if step_dir.parent else step_dir - ) - if metadata_info: - step_details.extend(metadata_info) + metadata_info = _format_caliper_metadata_info_for_step(get_file_link, step_dir) + step_details.extend(metadata_info) except Exception as e: logger.warning(f"Failed to extract caliper metadata for step {step_dir.name}: {e}") - # Extract censoring report for this specific step (before postprocess status) + # Extract censoring report for this specific step try: censoring_info = _format_censoring_report_info_for_step(step_dir, get_file_link) - if censoring_info: - step_details.extend(censoring_info) + step_details.extend(censoring_info) except Exception as e: logger.warning(f"Failed to extract censoring report for step {step_dir.name}: {e}") # Extract postprocess status for this specific step try: postprocess_info = _extract_postprocess_status_info(step_dir) - if postprocess_info: - step_details.extend(postprocess_info) + step_details.extend(postprocess_info) except Exception as e: logger.warning(f"Failed to extract postprocess status for step {step_dir.name}: {e}") @@ -1552,17 +1550,14 @@ def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) - censored_files = report_data.get("censored_files", 0) # Unexpected censoring # Create link to censoring report file - if get_file_link: - try: - report_link = get_file_link(censoring_report_path) - censoring_lines.append(f"* 🔒 [Censoring Report]({report_link})") - except Exception as e: - logger.warning( - f"Failed to create link for censoring report {censoring_report_path}: {e}" - ) - censoring_lines.append("* 🔒 Censoring Report") - else: - censoring_lines.append("* 🔒 Censoring Report") + try: + censoring_report_link = get_file_link(censoring_report_path, text="Censoring Report") + censoring_lines.append(f"* 🔒 {censoring_report_link}") + except Exception as e: + logger.warning( + f"Failed to create link for censoring report {censoring_report_path}: {e}" + ) + censoring_lines.append(f"* 🔒 Censoring Report **link generation failed** `{e}`") censoring_lines.append(f" * 📊 {total_files} files scanned") @@ -1592,13 +1587,10 @@ def _format_censoring_report_info_for_step(step_dir: Path, get_file_link: Any) - return [f"* 🔒 Censoring Report: Error reading report - `{e}`"] -def _format_caliper_metadata_info_for_step( - metadata_files: list[Path], get_file_link: Any, base_dir: Path -) -> list[str]: +def _format_caliper_metadata_info_for_step(get_file_link: Any, step_dir: Path) -> list[str]: """Format caliper metadata information for integration within step details.""" - if not metadata_files: - return [] + metadata_files = _search_caliper_metadata_files(step_dir) metadata_lines = [] for metadata_file in metadata_files: @@ -1611,7 +1603,7 @@ def _format_caliper_metadata_info_for_step( # Get relative path from base directory try: - relative_path = metadata_file.parent.relative_to(base_dir) + relative_path = metadata_file.parent.relative_to(step_dir) display_path = str(relative_path) if str(relative_path) != "." else "root" except ValueError: # If relative_to fails, use the full path @@ -1623,11 +1615,8 @@ def _format_caliper_metadata_info_for_step( timing = metadata.timing # Format path with link to metadata file - if get_file_link: - metadata_file_link = get_file_link(metadata_file) - path_info = f" * 📊 Test directory: [`{display_path}`]({metadata_file_link})" - else: - path_info = f" * 📊 Test directory: `{display_path}`" + metadata_file_link = get_file_link(metadata_file, text=f"`{display_path}`") + path_info = f"* 📊 **Test directory**: {metadata_file_link}" metadata_lines.append(path_info) # Look for completion information @@ -1662,11 +1651,9 @@ def _format_caliper_metadata_info_for_step( except Exception as e: logger.warning(f"Failed to process caliper metadata file {metadata_file}: {e}") - relative_path = ( - metadata_file.parent.relative_to(base_dir) if base_dir else metadata_file.parent - ) + relative_path = metadata_file.parent.relative_to(step_dir) metadata_lines.append( - f" * 📊 Test directory: `{relative_path}` - Error reading metadata: {e}" + f"* 📊 Test directory: `{relative_path}` - Error reading metadata: {e}" ) return metadata_lines From 9cd46cb54a77a1c8b565429528e198439ccb4770 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 09:24:58 +0200 Subject: [PATCH 0862/1072] [caliper/guidellm] optimize the report HTML embedded --- .../helpers/visualization_utils.py | 7 ++++-- .../guidellm/plotting/performance_analysis.py | 23 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/projects/caliper/postprocess/helpers/visualization_utils.py b/projects/caliper/postprocess/helpers/visualization_utils.py index 4531a8627..6feef4cee 100644 --- a/projects/caliper/postprocess/helpers/visualization_utils.py +++ b/projects/caliper/postprocess/helpers/visualization_utils.py @@ -99,8 +99,11 @@ def save_figure( width=None, # Remove any fixed width height=None, # Remove any fixed height ) - fig.write_html(output_file) - logger.debug(f"{final_filename} HTML saved successfully") + + # Use CDN for both standalone files and reports - simple and works + fig.write_html(output_file, include_plotlyjs="cdn") + logger.debug(f"{final_filename} HTML saved successfully (CDN optimized)") + return str(output_file) except Exception as e: diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index c777c0677..c32705f61 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -59,42 +59,43 @@ def _image_to_base64(image_path: str | Path) -> str: def _read_html_content(html_path: str | Path) -> str: - """Read HTML file content for embedding. + """Read HTML file content for report inclusion. + + Now that we use CDN for all files, we can directly read the body content + which contains just the plot without the library (since library comes from CDN). Args: html_path: Path to the HTML file Returns: - HTML content string + Body content ready for embedding in reports """ try: - logger.debug(f"🔍 Reading HTML content from: {html_path}") + html_path = Path(html_path) + logger.debug(f"🔍 Reading CDN-optimized content from: {html_path}") - # Check if file exists - if not Path(html_path).exists(): + if not html_path.exists(): logger.warning(f"❌ HTML file not found: {html_path}") return "" with open(html_path, encoding="utf-8") as html_file: content = html_file.read() content_size_kb = len(content.encode("utf-8")) / 1024 - logger.debug(f" 📄 Read {content_size_kb:.1f} KB from HTML file") + logger.debug(f" 📄 Read {content_size_kb:.1f} KB from CDN-optimized HTML file") - # Import re for regex operations + # Extract body content (no library since it uses CDN) import re - # Extract body content to avoid HTML document conflicts but keep all scripts body_match = re.search(r"]*>(.*?)", content, re.DOTALL | re.IGNORECASE) if body_match: body_content = body_match.group(1) body_size_kb = len(body_content.encode("utf-8")) / 1024 - logger.debug(f" ✅ Extracted body content: {body_size_kb:.1f} KB") + logger.debug(f" ✅ Extracted body content: {body_size_kb:.1f} KB (CDN-optimized)") return body_content else: - # If no body tags found, use full content logger.debug(" ⚠️ No body tags found, using full content") - logger.debug(f" 📄 Using full HTML content: {content_size_kb:.1f} KB") return content + except Exception as e: logger.warning(f"❌ Failed to read HTML content from {html_path}: {e}") return "" From a12a9b56957cffc4fb131819b621286abc24af7e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 09:37:45 +0200 Subject: [PATCH 0863/1072] [fournos_launcher] add fjob.spec.TTL field --- .../orchestration/config.yaml | 1 + .../fournos_launcher/orchestration/pr_args.py | 24 +++++++ .../fournos_launcher/orchestration/submit.py | 1 + projects/fournos_launcher/toolbox/__init__.py | 4 ++ .../toolbox/submit_and_wait/main.py | 9 ++- .../submit_and_wait/templates/job.yaml.j2 | 3 + projects/fournos_launcher/toolbox/utils.py | 69 +++++++++++++++++++ 7 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 projects/fournos_launcher/toolbox/utils.py diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 5be5ca092..97d6cfb28 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -20,6 +20,7 @@ fournos: name: psap-forge-ci key: kubeconfig job: + ttl: 12h # Valid format: ^(?:(?:\d+d)?(?:\d+h)?(?:\d+m)?)+$ (e.g., "12h", "1d6h", "30m") owner: "FORGE/fournos-launcher" # updated to the PR owner if available display_name: null # will be set to the job project/args if null pipeline_name: forge-test-only diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index 270d83dd8..cdee239e1 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -195,6 +195,29 @@ def handle_pipeline_directive(line: str) -> dict[str, str]: return {"fournos.job.pipeline_name": pipeline_name} +def handle_ttl_directive(line: str) -> dict[str, str]: + """ + Handle /ttl directive for setting the Fjob TTL name. + + Format: /ttl duration + + Args: + line: The directive line + + Returns: + Dictionary with ttl configuration + + Raises: + ValueError: If ttl_value is empty + """ + ttl_value = line.removeprefix("/ttl ").strip() + + if not ttl_value: + raise ValueError(f"Invalid /ttl directive: ttl value cannot be empty in '{line}'") + + return {"fournos.job.ttl": ttl_value} + + def handle_gpu_directive(line: str) -> dict[str, str]: """ Handle /gpu directive for setting GPU hardware requirements. @@ -351,6 +374,7 @@ def get_fournos_directive_handlers() -> dict[str, callable]: "/clusterless": handle_clusterless_directive, "/fournos": handle_fournos_directive, "/pipeline": handle_pipeline_directive, + "/ttl": handle_ttl_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, "/replot.url": handle_replot_url_directive, diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index fbe5ac3b8..89aa6de09 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -306,6 +306,7 @@ def submit_job(): "clusterless": config.project.get_config("fournos.job.clusterless"), "gpu_count": gpu_count, "gpu_type": gpu_type, + "ttl": config.project.get_config("fournos.job.ttl"), } if parallel_job_configs: diff --git a/projects/fournos_launcher/toolbox/__init__.py b/projects/fournos_launcher/toolbox/__init__.py index 5817ebfdc..7b869a575 100644 --- a/projects/fournos_launcher/toolbox/__init__.py +++ b/projects/fournos_launcher/toolbox/__init__.py @@ -4,3 +4,7 @@ Commands for managing FOURNOS jobs including submission, monitoring, cleanup, and shutdown operations. FOURNOS provides batch job orchestration capabilities for running workloads in Kubernetes environments. """ + +from .utils import validate_ttl + +__all__ = ["validate_ttl"] diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index b9bc9e778..a859b7e28 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -23,6 +23,7 @@ from projects.core.dsl.control_flow import EarlyReturn from projects.core.dsl.utils.k8s import is_valid_k8s_name, sanitize_k8s_name from projects.core.library import env as env_mod +from projects.fournos_launcher.toolbox import validate_ttl logger = logging.getLogger(__name__) @@ -54,7 +55,7 @@ def run( args: list = None, variables_overrides: dict = None, job_name: str = "", - namespace: str = "fournos-jobs", + namespace: str = "psap-automation", owner: str = "", display_name: str = "", pipeline_name: str = "", @@ -66,6 +67,7 @@ def run( gpu_type: str = None, clusterless: bool = False, wait: bool = True, + ttl: str = "12h", ): """ Submit a FOURNOS job and wait for completion @@ -149,6 +151,11 @@ def validate_inputs(args, ctx): if not isinstance(args.env, dict): raise ValueError("env should be a dict") + if not validate_ttl(args.ttl): + raise ValueError( + f"Invalid TTL format: '{args.ttl}'. Expected format like '12h', '30m', '1d6h', etc." + ) + return "Inputs validated" diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 index 07a84ea9b..c901bcf95 100644 --- a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 +++ b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 @@ -8,6 +8,9 @@ metadata: ci-run: {{ args.ci_label }} {%- endif %} spec: + {% if args.ttl -%} + ttl: {{ args.ttl }} + {% endif %} owner: {{ args.owner }} displayName: {{ args.display_name }} {% if args.clusterless -%} diff --git a/projects/fournos_launcher/toolbox/utils.py b/projects/fournos_launcher/toolbox/utils.py new file mode 100644 index 000000000..850e02164 --- /dev/null +++ b/projects/fournos_launcher/toolbox/utils.py @@ -0,0 +1,69 @@ +""" +Utilities for the FOURNOS launcher toolbox. +""" + +import re + + +def validate_ttl(ttl: str) -> bool: + """ + Validate TTL (Time to Live) format for FOURNOS jobs. + + Accepts formats like: + - "30m" (30 minutes) + - "2h" (2 hours) + - "1d" (1 day) + - "1h30m" (1 hour 30 minutes) + - "2d12h" (2 days 12 hours) + + Args: + ttl: TTL string to validate + + Returns: + bool: True if valid, False otherwise + + Examples: + >>> validate_ttl("12h") + True + >>> validate_ttl("30m") + True + >>> validate_ttl("1d12h30m") + True + >>> validate_ttl("invalid") + False + """ + if not isinstance(ttl, str): + return False + + # TTL regex pattern: supports combinations of days, hours, minutes + # Pattern breakdown: + # ^ - start of string + # (?: - non-capturing group for full pattern + # (?:\d+d)? - optional: one or more digits followed by 'd' (days) + # (?:\d+h)? - optional: one or more digits followed by 'h' (hours) + # (?:\d+m)? - optional: one or more digits followed by 'm' (minutes) + # ) + # $ - end of string + # + # The + after the main group ensures at least one time unit is present + ttl_pattern = r"^(?:(?:\d+d)?(?:\d+h)?(?:\d+m)?)+$" + + if not re.match(ttl_pattern, ttl.strip()): + return False + + # Additional check: ensure at least one time unit is actually present + # (the regex allows empty string, but we need at least one unit) + units = re.findall(r"\d+[dhm]", ttl.strip()) + if not units: + return False + + # Check for duplicate units (e.g., "1h2h" should be invalid) + unit_types = [unit[-1] for unit in units] # Extract the unit type (d, h, m) + if len(unit_types) != len(set(unit_types)): + return False + + return True + + +# TTL validation regex pattern (for reference in config.yaml) +TTL_REGEX_PATTERN = r"^(?:(?:\d+d)?(?:\d+h)?(?:\d+m)?)+$" From 2d9ee8ed0559f1b74780281075e7c8d3ec1dfa45 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 09:43:47 +0200 Subject: [PATCH 0864/1072] [fournos_launcher] orchestration: update to set clusterless by default --- projects/fournos_launcher/orchestration/cli.py | 7 ------- projects/fournos_launcher/orchestration/config.yaml | 2 +- projects/fournos_launcher/orchestration/pr_args.py | 2 +- projects/fournos_launcher/orchestration/submit.py | 7 ------- projects/fournos_launcher/toolbox/submit_and_wait/main.py | 3 --- .../toolbox/submit_and_wait/templates/job.yaml.j2 | 2 +- 6 files changed, 3 insertions(+), 20 deletions(-) diff --git a/projects/fournos_launcher/orchestration/cli.py b/projects/fournos_launcher/orchestration/cli.py index ff3eda17d..f85a330b5 100755 --- a/projects/fournos_launcher/orchestration/cli.py +++ b/projects/fournos_launcher/orchestration/cli.py @@ -65,13 +65,6 @@ def submit(ctx, cluster, project, args, namespace, override, commit): "--cluster or cluster.name is mandatory (unless clusterless mode is enabled)" ) - # Validate clusterless and exclusive modes are not both enabled - exclusive_mode = config.project.get_config("fournos.job.exclusive") - if clusterless_mode and exclusive_mode: - raise ValueError( - "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" - ) - if cluster: logger.info(f"Using cluster {cluster}") else: diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 97d6cfb28..60c977ba7 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -26,7 +26,7 @@ fournos: pipeline_name: forge-test-only ci_label: null # will be generated from the CI job UID exclusive: true - clusterless: null + clusterless: true env: OPENSHIFT_CI_job: [JOB_TYPE, JOB_NAME, JOB_SPEC, OPENSHIFT_CI, JOB_NAME_SAFE, BUILD_ID] OPENSHIFT_CI_git_pr: [PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_REF, REPO_NAME, REPO_OWNER, PULL_BASE_SHA, JOB_NAME, PULL_TITLE, PULL_REFS, PULL_HEAD_REF] diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index cdee239e1..db7df43ca 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -94,7 +94,7 @@ def handle_cluster_directive(line: str) -> dict[str, str]: if not cluster_name: raise ValueError(f"Invalid /cluster directive: cluster name cannot be empty in '{line}'") - return {"cluster.name": cluster_name} + return {"cluster.name": cluster_name, "fournos.job.clusterless": False} def handle_exclusive_directive(line: str) -> dict[str, str]: diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index 89aa6de09..b05044a31 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -265,13 +265,6 @@ def submit_job(): "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" ) - # Validate clusterless and exclusive modes are not both enabled - exclusive_mode = config.project.get_config("fournos.job.exclusive") - if clusterless_mode and exclusive_mode: - raise ValueError( - "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" - ) - # Get GPU hardware configuration gpu_count = config.project.get_config("fournos.job.hardware.gpu_count") gpu_type = config.project.get_config("fournos.job.hardware.gpu_type") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index a859b7e28..f113b462e 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -136,9 +136,6 @@ def validate_inputs(args, ctx): if not args.cluster_name and not args.clusterless: raise ValueError("cluster_name is required unless clusterless mode is enabled") - if args.clusterless and args.exclusive: - raise ValueError("Clusterless mode and exclusive mode cannot both be enabled") - if not args.project: raise ValueError("project is required") diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 index c901bcf95..1ecf56fae 100644 --- a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 +++ b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 @@ -18,7 +18,7 @@ spec: {%- else -%} cluster: {{ args.cluster_name }} {%- endif %} - exclusive: {{ args.exclusive }} + exclusive: {% if args.clusterless -%}false{% else %}{{ args.exclusive }}{%- endif %} pipeline: {{ args.pipeline_name }} {% if args.gpu_count and args.gpu_type -%} hardware: From e072e139385949beaec70d9458d6ce4282ac70fd Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Tue, 15 Sep 2026 10:35:40 +0100 Subject: [PATCH 0865/1072] feat: Add benchconf as optional dependency Co-Authored-By: Claude Opus 4.6 Claude-Session: https://claude.ai/code/session_01R2qiwvY5erYCbSfy4exJaM --- projects/core/image/Containerfile | 1 + pyproject.toml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/projects/core/image/Containerfile b/projects/core/image/Containerfile index 200c61e53..c6df79781 100644 --- a/projects/core/image/Containerfile +++ b/projects/core/image/Containerfile @@ -57,6 +57,7 @@ RUN uv lock && \ uv pip install --system -e ".[caliper]" && \ uv pip install --system -e ".[matrix_benchmarking]" && \ uv pip install --system -e ".[agentic_review]" && \ + uv pip install --system -e ".[benchconf]" && \ uv cache clean --no-progress # Create non-root user for security diff --git a/pyproject.toml b/pyproject.toml index b4b000971..2d445e449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,9 @@ agentic_review = [ "langchain-openai", "langchain-core", ] +benchconf = [ + "benchconf @ git+https://github.com/openshift-psap/benchconf@main", +] [project.scripts] caliper = "projects.caliper.cli.main:run_cli" From 3d3992540f9a76f0dab1caf4942522a3463fefff Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:14:45 +0200 Subject: [PATCH 0866/1072] [core] library: export: fix the access to get_ci_metadata_dir --- projects/core/library/ci.py | 29 ++++++++++++------- projects/core/library/env.py | 2 +- projects/core/library/export.py | 10 +++---- projects/core/library/export_notifications.py | 19 +++++++----- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 3abea996d..0e1615d3d 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -8,7 +8,6 @@ import functools import logging -import pathlib import sys import traceback @@ -21,8 +20,12 @@ logger = logging.getLogger(__name__) +def get_ci_metadata_dir_location(): + return env.BASE_ARTIFACT_DIR / "000__ci_metadata" + + # CI metadata directory path -def get_ci_metadata_dir(base_ci_dir=None): +def get_ci_metadata_dir(base_ci_dir, any_level=False): """Get the CI metadata directory path. Args: @@ -34,15 +37,19 @@ def get_ci_metadata_dir(base_ci_dir=None): Raises: ValueError: If both base_ci_dir and env.BASE_ARTIFACT_DIR are None """ - if base_ci_dir is not None: - return pathlib.Path(base_ci_dir) / "000__ci_metadata" - if env.BASE_ARTIFACT_DIR is not None: - return env.BASE_ARTIFACT_DIR / "000__ci_metadata" + meta_dir = base_ci_dir / "000__ci_metadata" + if meta_dir.exists(): + return meta_dir + + if not any_level: + raise ValueError(f"Cannot determine CI metadata directory in {base_ci_dir}") + + rglob = list(base_ci_dir.rglob("000__ci_metadata")) + if not rglob: + raise ValueError(f"Cannot find CI metadata directory at any level of {base_ci_dir}") - raise ValueError( - "Cannot determine CI metadata directory: both base_ci_dir parameter and env.BASE_ARTIFACT_DIR are None" - ) + return rglob[0] class HelpfulGroup(click.Group): @@ -185,7 +192,7 @@ def wrapper(*args, **kwargs): # Save exit status to YAML file try: - metadata_dir = get_ci_metadata_dir() + metadata_dir = get_ci_metadata_dir_location() metadata_dir.mkdir(parents=True, exist_ok=True) exit_status_file = metadata_dir / "exit_status.yaml" @@ -225,7 +232,7 @@ def add_notification_file( Returns: Path to the created notification file, or None if creation failed """ - metadata_dir = get_ci_metadata_dir(base_ci_dir) + metadata_dir = get_ci_metadata_dir_location() notifications_dir = metadata_dir / "notifications" notifications_dir.mkdir(parents=True, exist_ok=True) diff --git a/projects/core/library/env.py b/projects/core/library/env.py index c6dbc84f8..1c0beaf3e 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -133,7 +133,7 @@ def init(daily_artifact_dir=False): # Ensure CI metadata directory exists (lazy import to avoid circular imports) from . import ci as ci_lib - ci_metadata_dir = ci_lib.get_ci_metadata_dir() + ci_metadata_dir = ci_lib.get_ci_metadata_dir_location() ci_metadata_dir.mkdir(parents=True, exist_ok=True) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 05c5b843d..b1a545d57 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -133,7 +133,7 @@ def _process_caliper_postprocess_status( continue # Check for job shutdown/abort status - shutdown_status = _check_job_shutdown_status() + shutdown_status = _check_job_shutdown_status(step_dir) if shutdown_status: # Add shutdown information to status data status_data["job_shutdown"] = shutdown_status @@ -351,7 +351,7 @@ def caliper_export_entrypoint( ) # Check for job shutdown/abort status and add to mock status - shutdown_status = _check_job_shutdown_status() + shutdown_status = _check_job_shutdown_status(artifact_dir) if shutdown_status: status.job_shutdown = JobShutdown.from_dict(shutdown_status) logger.info(f"Added job shutdown status to dry run mock status: {shutdown_status}") @@ -363,7 +363,7 @@ def caliper_export_entrypoint( ) # Check for job shutdown/abort status and add to main export status - shutdown_status = _check_job_shutdown_status() + shutdown_status = _check_job_shutdown_status(artifact_dir) if shutdown_status: status.job_shutdown = JobShutdown.from_dict(shutdown_status) logger.info(f"Added job shutdown status to main export status: {shutdown_status}") @@ -382,7 +382,7 @@ def caliper_export_entrypoint( # Create failure status for notification status = ExportStatus(success=False, final_status=f"failed: {e}") # Check for job shutdown/abort status and add to failure status - shutdown_status = _check_job_shutdown_status() + shutdown_status = _check_job_shutdown_status(artifact_dir) if shutdown_status: status.job_shutdown = JobShutdown.from_dict(shutdown_status) except Exception as e: @@ -391,7 +391,7 @@ def caliper_export_entrypoint( # Create failure status for notification status = ExportStatus(success=False, final_status=f"failed: {e}") # Check for job shutdown/abort status and add to failure status - shutdown_status = _check_job_shutdown_status() + shutdown_status = _check_job_shutdown_status(artifact_dir) if shutdown_status: status.job_shutdown = JobShutdown.from_dict(shutdown_status) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index e861eac1d..fd7954676 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -425,13 +425,16 @@ def send_notification( return notification_success -def _get_project_and_args(project: str) -> tuple[str, str]: +def _get_project_and_args(project: str, artifact_dir: Path | None) -> tuple[str, str]: """Extract project name and args from fournos job or config.""" fjob_project = project fjob_args_str = "" try: - metadata_dir = ci_lib.get_ci_metadata_dir() + metadata_dir = ci_lib.get_ci_metadata_dir(base_ci_dir=artifact_dir, any_level=True) + if not metadata_dir: + return fjob_project, fjob_args_str + fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" if not fournos_fjob_path.exists(): return fjob_project, fjob_args_str @@ -477,7 +480,7 @@ def _build_enhanced_notification( status: ExportStatus, ) -> tuple[str, bool]: """Build enhanced notification with fournos job config and artifact links.""" - fjob_project, fjob_args_str = _get_project_and_args(project) + fjob_project, fjob_args_str = _get_project_and_args(project, artifact_dir) success = status.success censoring_occurred = status.censoring_occurred @@ -513,7 +516,7 @@ def _build_enhanced_notification( shutdown_value = shutdown_status.shutdown_value or "Stop" notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") - execution_engine_config = _get_execution_engine_config() + execution_engine_config = _get_execution_engine_config(artifact_dir) if execution_engine_config: notification_parts += ["", "---"] notification_parts.append("**Execution Engine Configuration**") @@ -661,10 +664,10 @@ def _get_censoring_report_section(artifact_dir: Path | None) -> list[str] | None return [f"⚠️ **Censoring report parsing failed:** {e}"] -def _get_execution_engine_config() -> str | None: +def _get_execution_engine_config(artifact_dir: Path) -> str | None: """Read and format execution engine configuration.""" try: - metadata_dir = ci_lib.get_ci_metadata_dir() + metadata_dir = ci_lib.get_ci_metadata_dir(artifact_dir, any_level=True) fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" if not fournos_fjob_path.exists(): return f"* FournosJob not found at `{fournos_fjob_path}`" @@ -1257,10 +1260,10 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> return step_details -def _check_job_shutdown_status() -> dict[str, Any] | None: +def _check_job_shutdown_status(artifact_dir: Path) -> dict[str, Any] | None: """Check if the job has been aborted via spec.shutdown field.""" try: - metadata_dir = ci_lib.get_ci_metadata_dir() + metadata_dir = ci_lib.get_ci_metadata_dir(artifact_dir, any_level=True) fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" if not fournos_fjob_path.exists(): return None From b4582b22a87de2e051aaaac7d825482c9ebb1463 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:42:32 +0200 Subject: [PATCH 0867/1072] [core] library: export_notifications: fix the overall test status --- projects/core/library/export_notifications.py | 99 +++++++++++++++++-- 1 file changed, 90 insertions(+), 9 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index fd7954676..a324e1a8a 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -321,11 +321,22 @@ def send_notification( """ # Extract notification parameters from status object project = config.project.get_config("project.name") - finish_reason = _extract_finish_reason_from_status(status) + + # Check individual step exit statuses from exit_status.yaml files once + step_status = None + if artifact_dir: + try: + step_status = _get_overall_step_status(artifact_dir) + if step_status == StepStatus.FAILURE: + logger.info("Step failure detected from exit_status.yaml files") + except Exception as e: + logger.warning(f"Failed to check step exit statuses for notification: {e}") + + finish_reason = _extract_finish_reason_from_status(status, step_status) # Build enhanced notification with fournos job info and artifact links notification_status, notification_success = _build_enhanced_notification( - artifact_dir, project, finish_reason, status + artifact_dir, project, finish_reason, status, step_status ) # Apply censoring to notification content before sending @@ -461,11 +472,15 @@ def _get_project_and_args(project: str, artifact_dir: Path | None) -> tuple[str, return fjob_project, fjob_args_str -def _extract_finish_reason_from_status(status: ExportStatus) -> str: - """Extract finish reason from status.""" +def _extract_finish_reason_from_status( + status: ExportStatus, step_status: StepStatus | None = None +) -> str: + """Extract finish reason from status, including step exit status checking.""" if status.job_shutdown and status.job_shutdown.is_aborted: return "aborted" elif not status.success: + return "export failed" + elif step_status == StepStatus.FAILURE: return "failed" elif status.censoring_occurred: return "completed with censoring" @@ -478,6 +493,7 @@ def _build_enhanced_notification( project: str, finish_reason: str, status: ExportStatus, + step_status: StepStatus | None = None, ) -> tuple[str, bool]: """Build enhanced notification with fournos job config and artifact links.""" fjob_project, fjob_args_str = _get_project_and_args(project, artifact_dir) @@ -485,19 +501,41 @@ def _build_enhanced_notification( success = status.success censoring_occurred = status.censoring_occurred + # Override success if step failures detected + step_failure_detected = step_status == StepStatus.FAILURE + if step_failure_detected: + success = False # Override success if any step failed + logger.info( + "Step failure detected from exit_status.yaml files - overriding success to False" + ) + logger.info( - f"Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}'" + f"Building notification - success={success}, censoring_occurred={censoring_occurred}, finish_reason='{finish_reason}', step_failure_detected={step_failure_detected}" ) - + status_emoji = "�" + status_reason = "unknown" + status_what = "status unknown" # Determine status emoji with abort taking precedence if status.job_shutdown and status.job_shutdown.is_aborted: - status_emoji = "🛑" + status_emoji = "⛔" + status_reason = "user abort" + status_what = "was aborted" + elif step_failure_detected: + status_emoji = "❌" # Step failures take precedence over export success + status_reason = "pipeline step failure" + status_what = "failed" elif not success: status_emoji = "❌" + status_reason = "export failure" + status_what = "completed" elif censoring_occurred: status_emoji = "⚠️" + status_reason = "censoring detected" + status_what = "completed" else: status_emoji = "✅" + status_reason = "success" + status_what = "completed" # Add total duration to base status total_duration = _read_total_duration(artifact_dir) @@ -506,7 +544,7 @@ def _build_enhanced_notification( if fjob_args_str: fjob_args_str = f" {fjob_args_str}" - base_status = f"{status_emoji} **Execution of `{fjob_project}`{fjob_args_str}** {duration_suffix} {status_emoji}" + base_status = f"{status_emoji} **Execution of `{fjob_project}`{fjob_args_str}** {status_what} (`{status_reason}`) after {duration_suffix} {status_emoji}" notification_parts = [base_status] # Add job abort message right below overall status if applicable @@ -514,7 +552,7 @@ def _build_enhanced_notification( if shutdown_status and shutdown_status.is_aborted: notification_parts += ["", "---"] shutdown_value = shutdown_status.shutdown_value or "Stop" - notification_parts.append(f"🛑 **JOB ABORTED** - `spec.shutdown={shutdown_value}`") + notification_parts.append(f"⛔ **JOB ABORTED** - `spec.shutdown={shutdown_value}`") execution_engine_config = _get_execution_engine_config(artifact_dir) if execution_engine_config: @@ -1389,6 +1427,49 @@ def _get_overall_status_from_steps(artifact_dir: Path) -> str: return "🔴" # Error checking = red +def _get_overall_step_status(artifact_dir: Path) -> StepStatus: + """Check all step exit statuses and return overall status as StepStatus enum.""" + + try: + current_step_name = Path(env.BASE_ARTIFACT_DIR).name + + step_statuses = [] + + for step_dir in sorted(artifact_dir.iterdir()): + if not step_dir.is_dir(): + continue + if step_dir.name.startswith("."): + continue + + # Only check directories that have run.log (actual steps) + run_log = step_dir / "run.log" + if not run_log.exists(): + continue + + _emoji, status = _read_step_exit_status(step_dir, current_step_name) + step_statuses.append(status) + + # Check for postprocess warnings in this step (always check, regardless of exit status) + postprocess_status = _check_postprocess_warnings(step_dir) + step_statuses.append(postprocess_status) + + # Priority: failure > ongoing > warning > unknown > success + if StepStatus.FAILURE in step_statuses: + return StepStatus.FAILURE + elif StepStatus.WARNING in step_statuses: + return StepStatus.WARNING + elif StepStatus.UNKNOWN in step_statuses: + return StepStatus.UNKNOWN + elif StepStatus.ONGOING in step_statuses: + return StepStatus.ONGOING + else: + return StepStatus.SUCCESS + + except Exception as e: + logger.exception(f"Failed to check step statuses: {e}") + return StepStatus.FAILURE # Error checking = failure + + def _read_step_duration(step_dir: Path) -> str: """Read step duration from timing file.""" From 8d5be6f6c916897bbaeaa49467bf7864bdb6554b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:48:31 +0200 Subject: [PATCH 0868/1072] [skeleton] orchestration: add the ability to fail on demand --- projects/skeleton/orchestration/config.yaml | 1 + projects/skeleton/orchestration/presets.d/presets.yaml | 3 +++ projects/skeleton/orchestration/test_skeleton.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 1fd65120c..6b2051bc5 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -17,6 +17,7 @@ skeleton: deep_testing: false test: duration_seconds: 0 # Default test duration in seconds + fail: false # set to true to force the project to fail caliper: export: diff --git a/projects/skeleton/orchestration/presets.d/presets.yaml b/projects/skeleton/orchestration/presets.d/presets.yaml index 211f236a8..16c955bc5 100644 --- a/projects/skeleton/orchestration/presets.d/presets.yaml +++ b/projects/skeleton/orchestration/presets.d/presets.yaml @@ -19,3 +19,6 @@ long_test: agent: agentic.enabled: true + +fail: + skeleton.test.fail: true diff --git a/projects/skeleton/orchestration/test_skeleton.py b/projects/skeleton/orchestration/test_skeleton.py index 5fe3e1796..0cd6ddc90 100644 --- a/projects/skeleton/orchestration/test_skeleton.py +++ b/projects/skeleton/orchestration/test_skeleton.py @@ -174,6 +174,9 @@ def do_test(): logger.info("") logger.info(f"Fake test configuration:\n{yaml_cfg}") + if config.project.get_config("skeleton.test.fail"): + raise RuntimeError("Configuration requested a failure") + # Capture benchmark timing benchmark_start_time = get_iso_timestamp() try: From 53da14ff02ea1c94d3f122094ec08646ec7b02f0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:53:28 +0200 Subject: [PATCH 0869/1072] [caliper] orchestration: postprocess: lower the log level --- projects/caliper/orchestration/postprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ac2c7e1a5..cc1e83513 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -1389,7 +1389,7 @@ def _run_s3_export_step(self) -> None: ai_data_dir = env.ARTIFACT_DIR / ai_data_step["ai_data_dir"] logger.info(f"AI data directory set to: {ai_data_dir}") else: - logger.warning( + logger.info( f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" ) From b29c35036b95595365f674c17d4822d7f7077839 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:54:52 +0200 Subject: [PATCH 0870/1072] [rhaiis] orchestration: ci: lower the log level of the preflight NOOP --- projects/rhaiis/orchestration/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 3508cdefd..b5e2deff9 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -171,7 +171,7 @@ def post_cleanup(ctx): def preflight(ctx) -> int: """Preflight check phase - Validate that the cluster if ready for testing.""" - logger.warning("Nothing so far for the preflight check") + logger.info("Nothing so far for the preflight check") return 0 From 5415cafda50b66900b428aec33eef684632483fd Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 16:55:04 +0200 Subject: [PATCH 0871/1072] [skeleton] orchestration: ci: lower the log level of the preflight NOOP --- projects/skeleton/orchestration/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/skeleton/orchestration/ci.py b/projects/skeleton/orchestration/ci.py index d8efa8e5c..58fcd8d7c 100755 --- a/projects/skeleton/orchestration/ci.py +++ b/projects/skeleton/orchestration/ci.py @@ -97,7 +97,7 @@ def post_cleanup(ctx): def preflight(ctx) -> int: """Preflight check phase - Validate that the cluster if ready for testing.""" - logger.warning("Nothing so far for the preflight check") + logger.info("Nothing so far for the preflight check") return 0 From 91fe127957edc94220afcef45974e3058fe41a03 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 17:14:05 +0200 Subject: [PATCH 0872/1072] [core] library: export_notifications: fix the test directory duplication --- projects/core/library/export_notifications.py | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index a324e1a8a..1d5ee2144 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -1145,64 +1145,6 @@ def _process_notification_files(step_dir: Path) -> list[str]: return notifications_from_files -def _extract_test_metadata_info(artifact_dir: Path, mlflow_run_url: str | None = None) -> list[str]: - """Extract test execution information from the test metadata files. - - Args: - artifact_dir: Directory to search for __test_labels__.yaml files - mlflow_run_url: Optional MLflow run URL for creating links - - Returns: - List of formatted strings with test information (directory, labels, success, message) - """ - test_info_lines = [] - - # Search for the test medata files recursively - test_metadata_files = _search_caliper_metadata_files(artifact_dir) - - if not test_metadata_files: - return [] - - for test_metadata_file in test_metadata_files: - try: - with open(test_metadata_file, encoding="utf-8") as f: - test_metadata = yaml.safe_load(f) or {} - - # Extract directory relative to artifact_dir - use just the immediate directory name - relative_dir = test_metadata_file.parent.relative_to(artifact_dir) - dir_name = relative_dir.name if relative_dir != Path(".") else "root" - - # Extract completion info - completion = test_metadata.get("completion", {}) - success = completion.get("success") - message = completion.get("message") - - if message: - message = f": `{message}`" - else: - message = "" - - # Format status - if success: - status_emoji = "✅" - elif success is False: - status_emoji = "❌" - else: - status_emoji = "❓" - - # Create link to __test_labels__.yaml file if MLflow URL is available - dir_link = _create_mlflow_file_link( - f"**{dir_name}**", mlflow_run_url, test_metadata_file - ) - - test_info_lines.append(f"* {status_emoji} {dir_link}{message}") - - except Exception as e: - test_info_lines.append(f"**{test_metadata_file.name}**: Error reading file - {e}") - - return test_info_lines - - def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: """Extract post-processing status information from POSTPROCESS_STATUS_FILENAME files. @@ -1264,14 +1206,6 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> """Process test labels, caliper metadata, and postprocess status for a single step directory.""" step_details = [] - # Extract test labels for this specific step - try: - test_metadata_info = _extract_test_metadata_info(step_dir, mlflow_run_url) - if test_metadata_info: - step_details.extend(test_metadata_info) - except Exception as e: - logger.warning(f"Failed to extract test labels for step {step_dir.name}: {e}") - get_file_link = _create_get_file_link(mlflow_run_url) # Extract caliper metadata for this specific step @@ -1703,14 +1637,6 @@ def _format_caliper_metadata_info_for_step(get_file_link: Any, step_dir: Path) - path_info = f"* 📊 **Test directory**: {metadata_file_link}" metadata_lines.append(path_info) - # Look for completion information - if metadata.completion: - status_emoji = "✅" if metadata.completion.success else "❌" - message_part = ( - f" `{metadata.completion.message}`" if metadata.completion.message else "" - ) - metadata_lines.append(f" * {status_emoji}{message_part}") - # Format labels if labels: label_items = [f"`{k}={v}`" for k, v in labels.items()] From 8f42bf174a72b1084a7f7a07d76205c60f7101e8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 18:36:09 +0200 Subject: [PATCH 0873/1072] [core] library: export_notifications: fix the test title --- projects/core/library/export_notifications.py | 79 +++++++++++++------ 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 1d5ee2144..779f6f0f8 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -436,40 +436,48 @@ def send_notification( return notification_success -def _get_project_and_args(project: str, artifact_dir: Path | None) -> tuple[str, str]: - """Extract project name and args from fournos job or config.""" +def _get_project_and_args(project: str, artifact_dir: Path | None) -> tuple[str, str, str]: + """Extract project name, args, and job info from fournos job or config.""" fjob_project = project fjob_args_str = "" + job_info_line = "" try: metadata_dir = ci_lib.get_ci_metadata_dir(base_ci_dir=artifact_dir, any_level=True) if not metadata_dir: - return fjob_project, fjob_args_str + return fjob_project, fjob_args_str, job_info_line fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" if not fournos_fjob_path.exists(): - return fjob_project, fjob_args_str + return fjob_project, fjob_args_str, job_info_line with open(fournos_fjob_path, encoding="utf-8") as f: fjob_data = yaml.safe_load(f) - display_name = fjob_data.get("spec", {}).get("displayName", "") - if not display_name: - return fjob_project, fjob_args_str + # Get project name and args from executionEngine.forge configuration + spec = fjob_data.get("spec", {}) + execution_engine = spec.get("executionEngine", {}) + forge_config = execution_engine.get("forge", {}) + fjob_project = forge_config.get("project", project) - parts = display_name.split() - if not parts: - return fjob_project, fjob_args_str + # Get forge args from the executionEngine.forge.args + forge_args = forge_config.get("args", []) + if forge_args: + fjob_args_str = " ".join(forge_args) - fjob_project = parts[0] - fjob_args_str = " ".join(parts[1:]) if len(parts) > 1 else "" + # Get name and displayName for the second line + fjob_name = fjob_data.get("metadata", {}).get("name", "name not found") + display_name = spec.get("displayName", "displayName not set") + + job_info_line = f"> `{fjob_name}` -- _{display_name}_" + + logger.info( + f"Extracted fjob info - project: {fjob_project}, args: {fjob_args_str}, job_info: {job_info_line}" + ) except Exception as e: logger.warning(f"Failed to read fournos job for project/args: {e}") - if fjob_args_str: - fjob_args_str = f" | `{fjob_args_str}`" - - return fjob_project, fjob_args_str + return fjob_project, fjob_args_str, job_info_line def _extract_finish_reason_from_status( @@ -496,7 +504,7 @@ def _build_enhanced_notification( step_status: StepStatus | None = None, ) -> tuple[str, bool]: """Build enhanced notification with fournos job config and artifact links.""" - fjob_project, fjob_args_str = _get_project_and_args(project, artifact_dir) + fjob_project, fjob_args_str, job_info_line = _get_project_and_args(project, artifact_dir) success = status.success censoring_occurred = status.censoring_occurred @@ -534,19 +542,27 @@ def _build_enhanced_notification( status_what = "completed" else: status_emoji = "✅" - status_reason = "success" - status_what = "completed" + status_reason = None + status_what = "completed with success" # Add total duration to base status total_duration = _read_total_duration(artifact_dir) duration_suffix = f" `{total_duration}`" if total_duration else "" + # Format the execution line with project and args separated if fjob_args_str: - fjob_args_str = f" {fjob_args_str}" + execution_text = f"Execution of `{fjob_project}` | `{fjob_args_str}`" + else: + execution_text = f"Execution of `{fjob_project}`" - base_status = f"{status_emoji} **Execution of `{fjob_project}`{fjob_args_str}** {status_what} (`{status_reason}`) after {duration_suffix} {status_emoji}" + status_reason_str = f" (`{status_reason}`)" if status_reason else "" + base_status = f"{status_emoji} {execution_text} {status_what}{status_reason_str} after {duration_suffix} {status_emoji}" notification_parts = [base_status] + # Add job info line (name and displayName) if available + if job_info_line: + notification_parts.append(job_info_line) + # Add job abort message right below overall status if applicable shutdown_status = status.job_shutdown if shutdown_status and shutdown_status.is_aborted: @@ -1634,13 +1650,30 @@ def _format_caliper_metadata_info_for_step(get_file_link: Any, step_dir: Path) - # Format path with link to metadata file metadata_file_link = get_file_link(metadata_file, text=f"`{display_path}`") - path_info = f"* 📊 **Test directory**: {metadata_file_link}" + + # Add completion status emoji to the test directory line + completion_emoji = "📊" + if metadata.completion: + if metadata.completion.success: + pass # no change + elif metadata.completion.success is False: + completion_emoji = "❌" + else: + completion_emoji = "❓" + + path_info = ( + f"* {completion_emoji} **Test directory**: {metadata_file_link}{completion_emoji}" + ) metadata_lines.append(path_info) + # Add completion message as a separate line + if metadata.completion and metadata.completion.message: + metadata_lines.append(f" * `{metadata.completion.message}`") + # Format labels if labels: label_items = [f"`{k}={v}`" for k, v in labels.items()] - metadata_lines.append(f" * {', '.join(label_items)}") + metadata_lines.append(f" * {', '.join(label_items)}") # Format KPI labels if kpi_labels: From 1d62d4cb390e198ba4f24fddc76f1a44db6f0f4f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Tue, 15 Sep 2026 21:22:28 +0200 Subject: [PATCH 0874/1072] [core] library: export_notifications: adjust the post-processing status --- projects/caliper/orchestration/notification.py | 2 +- projects/caliper/tests/test_notification_integration.py | 8 ++++---- projects/core/library/export_notifications.py | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 6b03faa34..5caac102a 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -42,7 +42,7 @@ def postprocess_get_file_link(file_path: Path | str, text: str = None) -> str: return get_file_link(base_directory / file_path, text) - lines.append(f"**Post-processing Status** {status_emoji} `{base_directory}`") + lines.append(f"{status_emoji} **Post-processing Status** `{base_directory}`") # Convert steps from list[dict] format to sorted list of (step_name, step_data) tuples step_tuples = [] diff --git a/projects/caliper/tests/test_notification_integration.py b/projects/caliper/tests/test_notification_integration.py index 00ad8de1e..e439baebc 100644 --- a/projects/caliper/tests/test_notification_integration.py +++ b/projects/caliper/tests/test_notification_integration.py @@ -80,7 +80,7 @@ def mock_get_file_link(file_path, text: str = None): print(f"Generated notification text:\n{notification_text}\n") # Verify overall structure - assert "**Post-processing Status** ✅" in notification_text + assert "✅ **Post-processing Status**" in notification_text assert ( "- ✅ [**parse**](https://example.com/artifacts/logs/001_parse.log): `success`" in notification_text @@ -148,7 +148,7 @@ def mock_get_file_link(file_path, text: str = None): notification_text = format_postprocess_status_notification(status, mock_get_file_link) # Verify overall failure status - assert "**Post-processing Status** ❌" in notification_text + assert "❌ **Post-processing Status**" in notification_text # Verify parse success assert "- ✅ **parse**: `success`" in notification_text @@ -189,7 +189,7 @@ def test_notification_formatting_failure(): notification_text = format_postprocess_status_notification(status) # Verify overall failure status - assert "**Post-processing Status** ❌" in notification_text + assert "❌ **Post-processing Status**" in notification_text # Verify failed step assert "- ❌ **parse**: `failed`" in notification_text @@ -219,7 +219,7 @@ def test_notification_without_file_links(): notification_text = format_postprocess_status_notification(status) # Verify basic formatting without links - assert "**Post-processing Status** ✅" in notification_text + assert "✅ **Post-processing Status**" in notification_text assert "- ✅ **parse**: `success`" in notification_text # Should not contain any links assert "https://" not in notification_text diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 779f6f0f8..7747b6cb4 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -1661,9 +1661,7 @@ def _format_caliper_metadata_info_for_step(get_file_link: Any, step_dir: Path) - else: completion_emoji = "❓" - path_info = ( - f"* {completion_emoji} **Test directory**: {metadata_file_link}{completion_emoji}" - ) + path_info = f"* {completion_emoji} **Test directory**: {metadata_file_link}" metadata_lines.append(path_info) # Add completion message as a separate line From 67eb6b642f49cf520e001e35797d8db85ca50b27 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 16 Sep 2026 07:47:07 +0100 Subject: [PATCH 0875/1072] fix: Resolve kaleido version conflict breaking image builds --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2d445e449..8a3f8d4c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ matrix_benchmarking = [ "pandas", "dash", "plotly", - "kaleido>=1.0.0", + "kaleido==0.2.1", "fire", "pyyaml", "BeautifulSoup4==4.11.*", From 173b3d69f9027aea26234c54ada1b3cc4cb8e2eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:17:44 +0200 Subject: [PATCH 0876/1072] [matrix_benchmarking] Remove --- .../library/matbenchmark.py | 134 --- .../matrix_benchmarking/library/visualize.py | 706 -------------- .../matrix_benchmarking/subproject/.gitignore | 0 .../subproject/.gitmodules | 3 - .../matrix_benchmarking/subproject/PLUGINS.md | 158 ---- .../matrix_benchmarking/subproject/README.md | 230 ----- .../subproject/bin/matbench | 11 - .../matrix_benchmarking/analyze/__init__.py | 35 - .../analyze/method/hunter.py | 91 -- .../analyze/method/skeleton.py | 68 -- .../analyze/method/stdev.py | 223 ----- .../analyze/method/zscore.py | 71 -- .../matrix_benchmarking/analyze/report.py | 887 ------------------ .../matrix_benchmarking/analyze_lts.py | 127 --- .../matrix_benchmarking/benchmark.py | 103 -- .../matrix_benchmarking/cli_args.py | 208 ---- .../subproject/matrix_benchmarking/common.py | 245 ----- .../matrix_benchmarking/download.py | 176 ---- .../matrix_benchmarking/download_lts.py | 160 ---- .../downloading/__init__.py | 131 --- .../downloading/scrape/middleware_ci.py | 62 -- .../downloading/scrape/ocp_ci.py | 71 -- .../downloading/scrape/s3.py | 71 -- .../matrix_benchmarking/exec/common.py | 176 ---- .../matrix_benchmarking/exec/kube.py | 12 - .../matrix_benchmarking/exec/prom.py | 233 ----- .../generate_lts_schema.py | 147 --- .../subproject/matrix_benchmarking/main.py | 60 -- .../subproject/matrix_benchmarking/matrix.py | 318 ------- .../subproject/matrix_benchmarking/models.py | 275 ------ .../subproject/matrix_benchmarking/parse.py | 167 ---- .../matrix_benchmarking/parsing/prom.py | 100 -- .../matrix_benchmarking/plotting/__init__.py | 36 - .../plotting/table_stats.py | 571 ----------- .../plotting/ui/__init__.py | 600 ------------ .../plotting/ui/assets/bWLwgP.css | 416 -------- .../plotting/ui/assets/resizing_script.js | 20 - .../matrix_benchmarking/plotting/ui/report.py | 306 ------ .../matrix_benchmarking/plotting/ui/web.py | 117 --- .../matrix_benchmarking/store/__init__.py | 205 ---- .../matrix_benchmarking/store/prom_db.py | 247 ----- .../matrix_benchmarking/store/simple.py | 290 ------ .../matrix_benchmarking/upload_lts.py | 253 ----- .../matrix_benchmarking/visualize.py | 92 -- .../subproject/requirements-exec.txt | 3 - .../subproject/requirements.txt | 13 - .../subproject/utils/wdm.py | 174 ---- .../helpers/analyze/__init__.py | 70 -- .../visualizations/helpers/plotting/html.py | 63 -- .../helpers/plotting/kpi_plot.py | 219 ----- .../helpers/plotting/kpi_table.py | 187 ---- .../helpers/plotting/lts_documentation.py | 116 --- .../helpers/plotting/prom/__init__.py | 258 ----- .../helpers/plotting/prom/cpu_memory.py | 246 ----- .../visualizations/helpers/plotting/report.py | 95 -- .../helpers/plotting/styles_css.py | 414 -------- .../visualizations/helpers/plotting/units.py | 46 - .../visualizations/helpers/store/__init__.py | 319 ------- .../helpers/store/k8s_quantity.py | 79 -- .../visualizations/helpers/store/parsers.py | 429 --------- .../visualizations/helpers/store/prom.py | 719 -------------- 61 files changed, 12062 deletions(-) delete mode 100644 projects/matrix_benchmarking/library/matbenchmark.py delete mode 100755 projects/matrix_benchmarking/library/visualize.py delete mode 100644 projects/matrix_benchmarking/subproject/.gitignore delete mode 100644 projects/matrix_benchmarking/subproject/.gitmodules delete mode 100644 projects/matrix_benchmarking/subproject/PLUGINS.md delete mode 100644 projects/matrix_benchmarking/subproject/README.md delete mode 100755 projects/matrix_benchmarking/subproject/bin/matbench delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/__init__.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/hunter.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/skeleton.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/stdev.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/zscore.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/report.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/benchmark.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/cli_args.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/common.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/download.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/download_lts.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/__init__.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/middleware_ci.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/ocp_ci.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/s3.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/common.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/kube.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/prom.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/generate_lts_schema.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/main.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/matrix.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/models.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/parse.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/parsing/prom.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/__init__.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/table_stats.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/__init__.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/bWLwgP.css delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/resizing_script.js delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/report.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/web.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/store/__init__.py delete mode 100755 projects/matrix_benchmarking/subproject/matrix_benchmarking/store/prom_db.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/store/simple.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/upload_lts.py delete mode 100644 projects/matrix_benchmarking/subproject/matrix_benchmarking/visualize.py delete mode 100644 projects/matrix_benchmarking/subproject/requirements-exec.txt delete mode 100644 projects/matrix_benchmarking/subproject/requirements.txt delete mode 100755 projects/matrix_benchmarking/subproject/utils/wdm.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/analyze/__init__.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/html.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_plot.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_table.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/lts_documentation.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/prom/__init__.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/prom/cpu_memory.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/report.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/styles_css.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/plotting/units.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/store/__init__.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/store/k8s_quantity.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/store/parsers.py delete mode 100644 projects/matrix_benchmarking/visualizations/helpers/store/prom.py diff --git a/projects/matrix_benchmarking/library/matbenchmark.py b/projects/matrix_benchmarking/library/matbenchmark.py deleted file mode 100644 index 0737e9dd6..000000000 --- a/projects/matrix_benchmarking/library/matbenchmark.py +++ /dev/null @@ -1,134 +0,0 @@ -import functools -import json -import logging - -import yaml - -from projects.core.library import config, env, run - - -def _json_dumper(obj, strict=False): - import datetime - import pathlib - - if hasattr(obj, "toJSON"): - return obj.toJSON() - - elif hasattr(obj, "json"): - return obj.dict(by_alias=True) - - elif hasattr(obj, "__dict__"): - return obj.__dict__ - - if isinstance(obj, datetime.datetime): - return obj.isoformat() - - elif isinstance(obj, pathlib.Path): - return str(obj) - elif not strict: - return str(obj) - else: - raise RuntimeError(f"No default serializer for object of type {obj.__class__}: {obj}") - - -def prepare_benchmark_file( - path_tpl: str, - script_tpl: str, - stop_on_error: bool, - common_settings: dict, - test_files: dict, - expe_name: str = None, - benchmark_values: dict = None, - expe_to_run: dict = None, -): - json_benchmark_file = {} - json_benchmark_file["--path_tpl"] = path_tpl - json_benchmark_file["--script_tpl"] = script_tpl - json_benchmark_file["--remote_mode"] = False - json_benchmark_file["--stop_on_error"] = stop_on_error - json_benchmark_file["test_files"] = test_files - json_benchmark_file["common_settings"] = common_settings - if expe_to_run: - json_benchmark_file["--expe_to_run"] = list(expe_to_run.keys()) - json_benchmark_file["expe"] = expe_to_run - else: - json_benchmark_file["--expe_to_run"] = [expe_name] - json_benchmark_file[expe_name] = expe = {} - expe[expe_name] = benchmark_values - - return json_benchmark_file - - -def save_benchmark_file(benchmark_file_content, dirname=None, name="benchmark.yaml"): - if dirname is None: - dirname = env.ARTIFACT_DIR - - benchmark_file = dirname / name - - logging.info(f"Saving MatrixBenchmarking benchmark file into {benchmark_file} ...") - # properly convert all the field to string - benchmark_file_safe_content = json.loads( - json.dumps(benchmark_file_content, default=functools.partial(_json_dumper, strict=False)) - ) - - yaml_content = yaml.dump( - benchmark_file_safe_content, default_flow_style=False, sort_keys=False, width=1000 - ) - with open(benchmark_file, "w") as f: - print(yaml_content, file=f) - - return benchmark_file, yaml_content - - -def set_benchmark_args(benchmark_file, expe_name=None, results_dirname=None): - if results_dirname is None: - results_dirname = benchmark_file.parent - - args = {} - - args["--workload"] = config.project.get_config("matbench.workload") - args["--benchmark_file"] = str(benchmark_file) - args["--results_dirname"] = str(results_dirname) - if expe_name: - args["--expe_to_run"] = expe_name - - return args - - -def run_benchmark(args, dry_run=False): - logging.info(f"Running MatrixBenchmarking rehearsal with '{args['--benchmark_file']}'...") - - BENCHMARK_CMD_BASE = "matbench benchmark" - cmd_args = " ".join([f"{k}={v}" for k, v in args.items()]) - cmd = f"TOPSAIL_FROM_CONFIG_FILE={config.project.config_path} {BENCHMARK_CMD_BASE} {cmd_args}" - - with open(env.ARTIFACT_DIR / "benchmark.cmd", "w") as f: - print(cmd, file=f) - - rehearsal_log_file = env.ARTIFACT_DIR / "benchmark-rehearsal.log" - test_log_file = env.ARTIFACT_DIR / "benchmark.log" - try: - run.run(f"{cmd} 1>'{rehearsal_log_file}' 2>&1") - logging.info("Rehearsal done.") - except Exception: - logging.error( - f"MatrixBenchmark benchmark rehearsal failed. See '{rehearsal_log_file}' for further detals." - ) - return True # failed - - if dry_run: - logging.info("Dry-run enabled, skipping the benchmark execution.") - return - - try: - run.run(f"{cmd} --run 2>&1 | tee -a '{test_log_file}'") - logging.info("Benchmark done.") - except Exception: - msg = "MatrixBenchmark benchmark failed." - logging.error(msg) - with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: - print(msg, file=f) - - return True # failed - - return False # didn't fail diff --git a/projects/matrix_benchmarking/library/visualize.py b/projects/matrix_benchmarking/library/visualize.py deleted file mode 100755 index 2b4385e39..000000000 --- a/projects/matrix_benchmarking/library/visualize.py +++ /dev/null @@ -1,706 +0,0 @@ -#! /usr/bin/env python - -import functools -import logging -import os -import pathlib -import shutil -import subprocess -import sys - -import yaml - -logging.getLogger().setLevel(logging.INFO) - -import fire - -from projects.legacy.library import config, env, run - -TOPSAIL_DIR = pathlib.Path(config.__file__).parents[3] - -matbench_config = None # will be set in init() -matbench_workload = None # will be set in init() -workload_storage_dir = None # will be set in init() - - -def init(allow_no_config_file=False): - global matbench_config, matbench_workload, workload_storage_dir - - env.init() - config_file = os.environ.get("TOPSAIL_FROM_CONFIG_FILE") - if not config_file and not allow_no_config_file: - raise RuntimeError( - "TOPSAIL_FROM_CONFIG_FILE must be set. Please source your `configure.sh` before running this file." - ) - - if not config_file: - logging.info( - "Running without TOPSAIL_FROM_CONFIG_FILE, skipping most of the initialization" - ) - return - - config.init(pathlib.Path(config_file).parent) - - is_plot_test = config.project.get_config("PR_POSITIONAL_ARG_0", "", warn=False).endswith( - "-plot" - ) - if is_plot_test: - pr_arg_1 = config.project.get_config("PR_POSITIONAL_ARG_1", "") - if not pr_arg_1 and not config.project.get_config("matbench.preset"): - raise ValueError("PR_POSITIONAL_ARG_1 or matbench.preset should have been set ...") - - # remove the matbench preset from the main config and store it in matbench.preset - config.project.set_config("PR_POSITIONAL_ARG_1", None) - config.project.set_config("matbench.preset", pr_arg_1) - - config.project.apply_preset_from_pr_args() - config.project.apply_config_overrides() - - matbench_workload = config.project.get_config("matbench.workload") - - workload_storage_dir = pathlib.Path(matbench_workload.replace(".", "/")) - - matbench_preset = config.project.get_config("matbench.preset") - if not matbench_preset: - pass # no preset defined, nothing to do - elif "://" in str(matbench_preset): - config.project.set_config("matbench.download.url", matbench_preset) - else: - config.project.set_config("matbench.config_file", f"{matbench_preset}.yaml") - config.project.set_config( - "matbench.download.url_file", workload_storage_dir / "data" / f"{matbench_preset}.yaml" - ) - - matbench_config = config.Config( - workload_storage_dir, - workload_storage_dir / "data" / config.project.get_config("matbench.config_file"), - ) - - -def entrypoint(allow_no_config_file=False): - def decorator(fct): - @functools.wraps(fct) - def wrapper(*args, **kwargs): - init(allow_no_config_file) - fct(*args, **kwargs) - - return wrapper - - return decorator - - -@entrypoint(allow_no_config_file=True) -def prepare_matbench(): - run.run(f""" - pip install --quiet --requirement "{TOPSAIL_DIR}/projects/matrix_benchmarking/subproject/requirements.txt" numpy==1.26.4 certifi==2023.5.7 - """) - - -@entrypoint() -def generate_visualizations(results_dirname, generate_lts=None, test_failed=None): - visualizations = matbench_config.get_config("visualize") - - for idx in range(len(visualizations)): - generate_visualization( - results_dirname, idx, generate_lts=generate_lts, test_failed=test_failed - ) - - -def call_parse(step_idx, common_args, common_env): - parse_env = common_env.copy() - parse_args = common_args.copy() - - mode = config.project.get_config("matbench.download.mode", None) - if mode != "prefer_cache": - logging.info(f"Download mode set to '{mode}', ignoring the parser cache.") - parse_env["MATBENCH_STORE_IGNORE_CACHE"] = "y" - - parse_args["output-matrix"] = env.ARTIFACT_DIR / "internal_matrix.json" - - parse_args_str = " ".join(f"'--{k}={v}'" for k, v in parse_args.items()) - parse_env_str = "env " + " ".join(f"'{k}={v}'" for k, v in parse_env.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_parse.log" - - cmd = f"{parse_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench parse {parse_args_str} |& tee > {log_file}" - - non_fatal_errors = [] - fatal_errors = [] - if run.run(cmd, check=False).returncode != 0: - logging.warning("An error happened while parsing the results ...") - fatal_errors.append(log_file.name) - - if log_has_errors(log_file): - logging.warning("Errors detected in the log file. Not aborting the processing.") - non_fatal_errors.append(log_file.name) - - return fatal_errors, non_fatal_errors - - -def call_generate_lts(step_idx, common_args, common_env_str): - lts_args = common_args.copy() - lts_args["output-lts"] = env.ARTIFACT_DIR / "lts_payload.json" - lts_args_str = " ".join(f"'--{k}={v}'" for k, v in lts_args.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_generate_lts.log" - - cmd = f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench parse {lts_args_str} |& tee > {log_file}" - - errors = [] - if run.run(cmd, check=False).returncode != 0: - logging.warning("An error happened while generating the LTS payload ...") - errors.append(log_file.name) - - if log_has_errors(log_file): - errors.append(log_file.name) - - return errors - - -def call_generate_lts_schema(step_idx, common_args): - lts_schema_args = common_args.copy() - - lts_schema_args.pop("results_dirname") - lts_schema_args["file"] = env.ARTIFACT_DIR / "lts_payload.schema.json" - lts_schema_args_str = " ".join(f"'--{k}={v}'" for k, v in lts_schema_args.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_generate_lts_schema.log" - - cmd = f"/app/forge/projects/matrix_benchmarking/subproject/bin/matbench generate_lts_schema {lts_schema_args_str} |& tee > {log_file}" - - errors = [] - if run.run(cmd, check=False).returncode != 0: - logging.warning("An error happened while generating the LTS payload schema...") - errors.append(log_file.name) - - if log_has_errors(log_file): - errors.append(log_file.name) - - return errors - - -def generate_opensearch_config_yaml_env(dest=None): - instance = config.project.get_config("matbench.lts.opensearch.instance") - - _index = config.project.get_config("matbench.lts.opensearch.index") - index_prefix = config.project.get_config("matbench.lts.opensearch.index_prefix") or "" - index = f"{index_prefix}{_index}" - - vault_key = config.project.get_config("secrets.dir.env_key") - opensearch_instances_file = config.project.get_config("secrets.opensearch_instances") - - with open(pathlib.Path(os.environ[vault_key]) / opensearch_instances_file) as f: - instances_file_doc = yaml.safe_load(f) - - env_doc = dict( - opensearch_username=instances_file_doc[instance]["username"], - opensearch_password=instances_file_doc[instance]["password"], - opensearch_port=instances_file_doc[instance]["port"], - opensearch_host=instances_file_doc[instance]["host"], - opensearch_index=index, - instance=instance, - ) - - if dest is None: - return env_doc - - with open(dest, "w") as f: - yaml.dump(env_doc, f, indent=4) - - -def get_lts_directory(common_args): - workload = common_args["workload"] - results_dirname = pathlib.Path(common_args["results_dirname"]) - - return results_dirname / "lts" / workload - - -def call_download_lts(step_idx, common_args, common_env_str): - download_args = common_args.copy() - - download_args["lts_results_dirname"] = get_lts_directory(common_args) - - download_args.pop("results_dirname") - download_args.pop("workload") - download_args.pop("workload_base_dir") - download_args["force"] = True - download_args["clean"] = True - - download_args_str = " ".join(f"'--{k}={v}'" for k, v in download_args.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_download_lts.log" - - cmd = f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench download_lts {download_args_str} |& tee > {log_file}" - - errors = [] - if (err_code := run.run(cmd, check=False).returncode) != 0: - if err_code == 123: # No medium found - logging.warning("Couldn't download any LTS record, the index does not exist ...") - # not a fatal error, don't append it to `errors` - else: - logging.warning("An error happened while downloading the LTS payload ...") - errors.append(log_file.name) - - return errors - - -def call_upload_lts(step_idx, common_args, common_env_str): - upload_args = common_args.copy() - upload_args_str = " ".join(f"'--{k}={v}'" for k, v in upload_args.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_upload_lts.log" - - env_file = pathlib.Path(".env.generated.yaml") - generate_opensearch_config_yaml_env(env_file) - - cmd = f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench upload_lts {upload_args_str} |& tee > {log_file}" - - errors = [] - if run.run(cmd, check=False).returncode != 0: - logging.warning("An error happened while uploading the LTS payload ...") - errors.append(log_file.name) - - env_file.unlink() - - return errors - - -def call_analyze_lts(step_idx, common_args, common_env_str): - analyze_args = common_args.copy() - analyze_args["lts_results_dirname"] = get_lts_directory(common_args) - analyze_args_str = " ".join(f"'--{k}={v}'" for k, v in analyze_args.items()) - - log_file = env.ARTIFACT_DIR / f"{step_idx}_matbench_analyze_lts.log" - - env_file = pathlib.Path(".env.generated.yaml") - generate_opensearch_config_yaml_env(env_file) - - cmd = f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench analyze_lts {analyze_args_str} |& tee > {log_file}" - - errors = [] - retcode = run.run(cmd, check=False, cwd=env.ARTIFACT_DIR).returncode - - if retcode == 0: - logging.info("The regression analyses did not detect any regression.") - regression_detected = False - elif retcode < 100: - logging.warning("A regression was detected.") - regression_detected = "regression detected" - else: - logging.warning("An error happened while analyzing the LTS payload ...") - errors.append(log_file.name) - regression_detected = None # use the errors list - - env_file.unlink() - - regression_summary_path = env.ARTIFACT_DIR / "regression_summary.yaml" - - run.run_toolbox( - "repo", - "send_cpt_notification", - regression_summary_path=regression_summary_path, - title=config.project.get_config("matbench.lts.regression_analyses.notification.title"), - ) - - if regression_detected and regression_summary_path.exists(): - with open(regression_summary_path) as f: - regression_summary = yaml.safe_load(f) - - if regression_summary.get("significant_performance_increase", 0): - regression_detected = "performance boost" - else: - regression_detected = "performance regression" - - return errors, regression_detected - - -def log_has_errors(log_file): - has_errors = False - if not log_file.exists(): - logging.error(f"Log file {log_file} does not exist...") - return True - - with open(log_file) as log_f: - for line in log_f.readlines(): - has_errors |= line.startswith("ERROR") - has_errors |= line.startswith("FAILED:") - has_errors |= line.startswith("Traceback (most recent call last)") - if not has_errors: - continue - - with open(env.ARTIFACT_DIR / "FAILURE.txt", "a") as fail_f: - print(line, end="", file=fail_f) - logging.error(line.strip()) - - return has_errors - - -def call_visualize(step_idx, common_env_str, common_args, filters_to_apply, generate_url): - visu_args = common_args.copy() - visu_args["filters"] = filters_to_apply - visu_args["generate"] = generate_url - visu_args_str = " ".join(f"'--{k}={v}'" for k, v in visu_args.items()) - - dest_dir = env.ARTIFACT_DIR / filters_to_apply - dest_dir.mkdir(parents=True, exist_ok=True) - - log_file = dest_dir / f"{step_idx}_matbench_visualize.log" - - cmd = f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench visualize {visu_args_str} |& tee > {log_file}" - - errors = [] - if run.run(cmd, check=False, cwd=dest_dir).returncode != 0: - logging.warning("An error happened while generating the visualization ...") - errors.append(log_file.name) - - if log_has_errors(log_file): - errors.append(log_file.name) - - run.run(f""" - mkdir -p {dest_dir}/figures_{{png,html}} - mv {dest_dir}/fig_*.png "{dest_dir}/figures_png" 2>/dev/null || true - mv {dest_dir}/fig_*.html "{dest_dir}/figures_html" 2>/dev/null || true - """) - - return errors - - -def generate_visualization( - results_dirname, idx, generate_lts=None, upload_lts=None, analyze_lts=None, test_failed=None -): - generate_list = matbench_config.get_config(f"visualize[{idx}].generate") - if not generate_list: - raise ValueError(f"Couldn't get the configuration #{idx} ...") - - generate_url = "stats=" + "&stats=".join(generate_list) - - common_args, common_env = get_common_matbench_args_env(results_dirname) - common_env_str = "env " + " ".join(f"'{k}={v}'" for k, v in common_env.items()) - - step_idx = 0 - non_fatal_errors = [] - - # - # Parse the results, to validate that they are well formed - # - - fatal_errors, _non_fatal_errors = call_parse(step_idx, common_args, common_env) - non_fatal_errors += _non_fatal_errors - - if fatal_errors: - msg = f"A fatal error happened during the results parsing, aborting the visualization ({', '.join(fatal_errors)})." - with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: - print(msg, file=f) - logging.error(msg) - raise RuntimeError(msg) - - # - # Save the LTS in the artifacts, to have it along with the other files - # Save the LTS schema in the artifacts - # - - do_generate_lts = ( - generate_lts - if generate_lts is not None - else config.project.get_config("matbench.lts.generate", None) - ) - - if do_generate_lts: - step_idx += 1 - non_fatal_errors += call_generate_lts(step_idx, common_args, common_env_str) - - step_idx += 1 - non_fatal_errors += call_generate_lts_schema(step_idx, common_args) - - if config.project.get_config("matbench.download.save_to_artifacts", None): - shutil.copytree(common_args["MATBENCH_RESULTS_DIRNAME"], env.ARTIFACT_DIR / "downloaded") - - # - # Download the historical LTS payloads for comparions - # Analyze the new results for regression against the historical results - # - - replotting = config.project.get_config("PR_POSITIONAL_ARG_0", "").endswith("-plot") - - do_analyze_lts = ( - analyze_lts - if analyze_lts is not None - else config.project.get_config("matbench.lts.regression_analyses.enabled", None) - ) - - if replotting and not config.project.get_config( - "matbench.lts.regression_analyses.enabled_on_replot", None - ): - do_analyze_lts = False - - do_upload_lts = ( - upload_lts - if upload_lts is not None - else config.project.get_config("matbench.lts.opensearch.export.enabled", None) - ) - - if replotting and not config.project.get_config( - "matbench.lts.opensearch.export.enabled_on_replot", None - ): - do_upload_lts = False - - if do_analyze_lts or do_upload_lts: - try: - env_file = pathlib.Path(".env.generated.yaml") - generate_opensearch_config_yaml_env(env_file) - except Exception as e: - logging.warning(f"Couldn't generate OpenSearch credential file: {e}") - do_analyze_lts = False - do_upload_lts = False - - fail_on_regression = config.project.get_config( - "matbench.lts.regression_analyses.fail_test_on_regression" - ) - fail_on_upload = config.project.get_config( - "matbench.lts.opensearch.export.fail_test_on_fail" - ) - if fail_on_upload or fail_on_regression: - non_fatal_errors += ["opensearch secret cannot be generated"] - - if do_analyze_lts: - step_idx += 1 - download_lts_errors = call_download_lts(step_idx, common_args, common_env_str) - non_fatal_errors += download_lts_errors - - step_idx += 1 - analyze_lts_errors, regression_detected = call_analyze_lts( - step_idx, common_args, common_env_str - ) - non_fatal_errors += analyze_lts_errors - - if regression_detected: - if config.project.get_config( - "matbench.lts.regression_analyses.fail_test_on_regression" - ): - non_fatal_errors += [regression_detected] - else: - logging.warning( - "A regression has been detected, but ignored as per matbench.lts.regression_analyses.fail_test_on_regression" - ) - - # - # Upload the LTS payload for future regression analyses - # - - if do_upload_lts: - if config.project.get_config( - "matbench.lts.regression_analyses.upload_lts_on_regression", True - ): - skip_upload = ( - non_fatal_errors - and list(non_fatal_errors) != ["regression detected"] - and list(non_fatal_errors) != ["performance regression"] - and list(non_fatal_errors) != ["performance boost"] - ) - else: - # don't skip the upload if a regression has been detected - skip_upload = bool(non_fatal_errors) - - if not skip_upload: - step_idx += 1 - upload_lts_errors = call_upload_lts(step_idx, common_args, common_env_str) - - if config.project.get_config("matbench.lts.opensearch.export.fail_test_on_fail"): - non_fatal_errors += upload_lts_errors - elif upload_lts_errors: - logging.warning( - "An error happened during the LTS upload. Ignoring as per matbench.lts.opensearch.export.fail_test_on_fail." - ) - else: - logging.error( - "Errors have been detected in the post-processing. NOT uploading LTS payload to OpenSearch." - ) - - # - # Generate the visualization reports - # - - filters = matbench_config.get_config(f"visualize[{idx}]").get("filters", [None]) - for filters_to_apply in filters: - if not filters_to_apply: - filters_to_apply = "" - - step_idx += 1 - non_fatal_errors += call_visualize( - step_idx, common_env_str, common_args, filters_to_apply, generate_url - ) - - # - # Done :) - # - - if non_fatal_errors: - msg = f"An error happened during the visualization post-processing ... ({', '.join(non_fatal_errors)} in {env.ARTIFACT_DIR})" - if test_failed: - msg += ". Mind that the test that was processed FAILED." - elif test_failed is False: - msg += ". The test that was processed SUCCEEDED." - # if None, ignore - - logging.error(msg) - with open(env.ARTIFACT_DIR / "FAILURE.txt", "w") as f: - print(msg, file=f) - raise RuntimeError(msg) - - -@entrypoint() -def generate_from_dir(results_dirname, generate_lts=None, test_failed=None): - logging.info(f"Generating the visualization from '{results_dirname}' ...") - - generate_visualizations(results_dirname, generate_lts=generate_lts, test_failed=test_failed) - - -def get_common_matbench_args_env(results_dirname): - common_args = dict() - common_args["results_dirname"] = results_dirname - common_args["workload"] = config.project.get_config("matbench.workload") - - common_args["workload_base_dir"] = str(TOPSAIL_DIR) - - common_env = dict() - common_env["MPLCONFIGDIR"] = "/tmp" - - return common_args, common_env - - -def download(results_dirname): - url = config.project.get_config("matbench.download.url", None) - url_file = config.project.get_config("matbench.download.url_file", None) - - if url: - with open(env.ARTIFACT_DIR / "source_url", "w") as f: - print(url, file=f) - - elif url_file: - shutil.copyfile(url_file, env.ARTIFACT_DIR / "source_url") - else: - raise ValueError("matbench.download.url or matbench.download.url_file must be specified") - - common_args, common_env = get_common_matbench_args_env(results_dirname) - - download_args = common_args.copy() - download_args |= dict( - mode=config.project.get_config("matbench.download.mode"), - do_download=True, - ) - - env_key = config.project.get_config("secrets.dir.env_key") - secret_path = os.environ.get(env_key) - secret_filename = config.project.get_config("secrets.aws_credentials", None) - - if not env_key or not secret_path or not secret_filename: - logging.warning( - f"secrets.dir.env_key or ${secret_path} not set or {secret_filename=}. Cannot set AWS_SHARED_CREDENTIALS_FILE" - ) - else: - os.environ["AWS_SHARED_CREDENTIALS_FILE"] = secret_path + "/.awscred" - - if url: - download_args["url"] = url - elif url_file: - download_args["url_file"] = url_file - - download_args_str = " ".join(f"'--{k}={v}'" for k, v in download_args.items()) - common_env_str = "env " + " ".join(f"'{k}={v}'" for k, v in common_env.items()) - - run.run( - f"{common_env_str} /app/forge/projects/matrix_benchmarking/subproject/bin/matbench download {download_args_str} |& tee > {env.ARTIFACT_DIR}/_matbench_download.log" - ) - - -# generates a unique (sequential) build identifier -# -# uses the OpenSearch configuration to store the build counter -# -# `build_identifier` should be a fully qualified name, like -# `topsail-v1.2.0`, so that the build index starts at 1, as expected -def get_next_build_index(build_identifier): - import opensearchpy - import opensearchpy.exceptions - - # warning: `env_doc` contains the OpenSearch secrets, don't print it - env_doc = generate_opensearch_config_yaml_env() - - client = opensearchpy.OpenSearch( - hosts=[{"host": env_doc["opensearch_host"], "port": env_doc["opensearch_port"]}], - timeout=60, - http_compress=True, - http_auth=(env_doc["opensearch_username"], env_doc["opensearch_password"]), - use_ssl=True, - verify_certs=False, - ssl_assert_hostname=False, - ssl_show_warn=False, - ) - - _index = config.project.get_config("matbench.lts.opensearch.build_counter_index") - index_prefix = config.project.get_config("matbench.lts.opensearch.index_prefix") or "" - index = f"{index_prefix}{_index}" - - try: - # Ensure the index exists - if not client.indices.exists(index): - client.indices.create(index, ignore=400) - except opensearchpy.exceptions.ConnectionError: - logging.error( - "Failed to connect to OpenSearch. " - f"Configured to use the '{env_doc['instance']}' instance." - ) - raise - - # generated by ChatGPT - def opensearch_increment_and_get_counter(): - response = client.update( - index=index, - id=build_identifier, - body={ - "script": {"source": "ctx._source.count += 1", "lang": "painless"}, - "upsert": {"count": 1}, - }, - refresh=True, # ensures updated doc is searchable immediately - _source=True, # request updated _source back - ) - return response.get("get", {}).get("_source", {}).get("count", None) - - return opensearch_increment_and_get_counter() - - -@entrypoint() -def download_and_generate_visualizations(results_dirname="/tmp/matrix_benchmarking_results"): - download(results_dirname) - generate_visualizations(results_dirname) - - -class Visualize: - """ - Commands for launching the visualization commands - """ - - def __init__(self): - self.prepare_matbench = prepare_matbench - self.generate_visualizations = generate_visualizations - self.generate_visualizations_from_dir = generate_from_dir - self.download_and_generate_visualizations = download_and_generate_visualizations - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - fire.Fire(Visualize()) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - logging.error(f"Command '{e.cmd}' failed --> {e.returncode}") - sys.exit(1) - except KeyboardInterrupt: - print() # empty line after ^C - logging.error("Interrupted.") - sys.exit(1) diff --git a/projects/matrix_benchmarking/subproject/.gitignore b/projects/matrix_benchmarking/subproject/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/matrix_benchmarking/subproject/.gitmodules b/projects/matrix_benchmarking/subproject/.gitmodules deleted file mode 100644 index c8a142f3f..000000000 --- a/projects/matrix_benchmarking/subproject/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/workload"] - path = src/workload - url = https://github.com/openshift-psap/matrix-benchmarking-plugins diff --git a/projects/matrix_benchmarking/subproject/PLUGINS.md b/projects/matrix_benchmarking/subproject/PLUGINS.md deleted file mode 100644 index e98a1dc68..000000000 --- a/projects/matrix_benchmarking/subproject/PLUGINS.md +++ /dev/null @@ -1,158 +0,0 @@ -# Matrix Benchmarking Visualization Plugins - -This document provides an overview of all available visualization plugins in the TOPSAIL project. Each project contains specialized visualization modules that handle data processing, analysis, and reporting for different benchmarking scenarios. - -## Plugin Architecture - -Each visualization plugin follows a consistent structure: -- **models/**: Data models and KPI definitions -- **store/**: Data storage parsers and Prometheus integrations -- **plotting/**: Visualization components and report generators -- **analyze/**: Analysis modules - -All plugins implement a `register()` function to register their components with the matrix benchmarking framework. - -## Available Visualization Plugins - -### Active Projects - -#### 1. Container Benchmarking -**Location**: `projects/container_bench/visualizations/benchmark/` -**Purpose**: Benchmarking container performance and resource utilization -**Components**: -- Comparison reports for different container configurations -- Performance metrics visualization -- Resource usage analysis - -#### 2. CRC Timing Analysis -**Location**: `projects/crc-timing/visualizations/crc-timing/` -**Purpose**: Analyzing CodeReady Containers startup and operational timing -**Components**: -- Initialization timing reports -- Error analysis and reporting -- Performance metrics tracking - -#### 3. Fine-Tuning Performance Analysis -**Location**: `projects/fine_tuning/visualizations/` -**Purpose**: Comprehensive analysis of model fine-tuning performance across different frameworks -**Modules**: -- `fine_tuning/`: Core fine-tuning metrics and analysis -- `fine_tuning_prom/`: Prometheus metrics integration for fine-tuning -- `fms_hf_tuning/`: Foundation Model Stack and HuggingFace tuning analysis -- `fms_prom/`: FMS Prometheus metrics -- `ibm_comparison/`: IBM model comparison reports -- `ilab_prom/`: InstructLab Prometheus metrics -- `ilab_training/`: InstructLab training analysis -- `ray_benchmark/`: Ray framework benchmarking -- `ray_prom/`: Ray Prometheus integration - -#### 4. KServe Model Serving -**Location**: `projects/kserve/visualizations/` -**Purpose**: Analysis of KServe model serving performance and scaling -**Modules**: -- `kserve-llm/`: Large Language Model serving analysis -- `kserve-prom/`: KServe Prometheus metrics -- `kserve-scale/`: Scaling performance analysis - -#### 5. Mac AI Load Testing -**Location**: `projects/mac_ai/visualizations/llm_load_test/` -**Purpose**: Load testing analysis for Large Language Model inference on Mac systems -**Components**: -- Latency and throughput analysis -- Token processing metrics -- GPU and CPU utilization tracking -- Llama benchmark integration - -#### 6. Matrix Benchmarking Helpers -**Location**: `projects/matrix_benchmarking/visualizations/helpers/` -**Purpose**: Shared visualization utilities and common plotting functions -**Components**: -- Common plotting utilities -- Report generation helpers -- Shared data processing functions - -#### 7. RHODS Pipelines -**Location**: `projects/pipelines/visualizations/rhods-pipelines/` -**Purpose**: Red Hat OpenShift Data Science Pipelines performance analysis -**Components**: -- Pipeline execution timeline analysis -- Pod-node mapping visualization -- Performance distribution reports -- Resource utilization tracking - -#### 8. Skeleton Template -**Location**: `projects/skeleton/visualizations/skeleton/` -**Purpose**: Template and example visualization plugin for new projects -**Components**: -- Basic report structure -- Prometheus integration example -- Error reporting template -- Control plane monitoring examples - -### Deprecated Projects - -The following projects contain legacy visualization plugins that are no longer actively maintained but may still be referenced: - -- `projects/deprecated/busy_cluster/visualizations/` -- `projects/deprecated/codeflare/visualizations/` -- `projects/deprecated/load-aware/visualizations/` -- `projects/deprecated/notebooks/visualizations/` -- `projects/deprecated/scheduler/visualizations/` - -## Plugin Registration - -Each visualization plugin registers its components through the `register()` function in the main `__init__.py` file. This function typically registers: - -- Report generators -- Data parsers -- Prometheus metric collectors -- Analysis modules -- KPI tables and plots -- LTS (Long Term Storage) documentation - -## Common Components - -Most plugins include these standard components: - -### Models -- **KPI Models**: Key Performance Indicator definitions -- **LTS Models**: Long Term Storage schema definitions -- **Data Models**: Pydantic models for data validation - -### Store -- **Parsers**: Raw data parsing and transformation -- **Prometheus Integration**: Metrics collection and processing -- **LTS Parser**: Long-term storage data handling - -### Plotting -- **Reports**: Dashboard-style comprehensive reports -- **Error Reports**: Error analysis and visualization -- **Metrics Plots**: Individual metric visualizations -- **Comparison Reports**: Side-by-side analysis - -### Analysis -- **Statistical Analysis**: Performance trend analysis -- **Regression Detection**: Performance regression identification -- **Anomaly Detection**: Unusual behavior identification - -## Usage - -To use a visualization plugin: - -1. Ensure the plugin is registered in the matrix benchmarking framework -2. Configure the appropriate data sources and metrics -3. Run the visualization generation through the matrix benchmarking CLI -4. Access generated reports through the web interface - -## Development - -When creating a new visualization plugin: - -1. Use the `skeleton` project as a template -2. Follow the standard directory structure -3. Implement the required `register()` function -4. Define appropriate data models -5. Create plotting and analysis components -6. Test with sample data - -For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. diff --git a/projects/matrix_benchmarking/subproject/README.md b/projects/matrix_benchmarking/subproject/README.md deleted file mode 100644 index b42a5a0fe..000000000 --- a/projects/matrix_benchmarking/subproject/README.md +++ /dev/null @@ -1,230 +0,0 @@ -# Matrix Benchmarking - -_(documentation generated by Claude Code)_ - -A comprehensive benchmarking and performance analysis framework designed for OpenShift/Kubernetes environments. Matrix Benchmarking provides tools for downloading, parsing, analyzing, and visualizing performance data with support for long-term storage and advanced statistical analysis. - -## Features - -### Core Functionality -- **Benchmark Execution**: Run performance benchmarks in Kubernetes/OpenShift environments -- **Data Collection**: Download and scrape benchmark data from multiple sources -- **Result Parsing**: Parse Prometheus metrics and other benchmark outputs -- **Data Analysis**: Statistical analysis with multiple methods (standard deviation, Z-score, Hunter anomaly detection) -- **Visualization**: Generate interactive web-based reports and visualizations -- **Long-term Storage**: Schema generation and data management for historical analysis - -### Supported Data Sources -- **Middleware CI**: Scrape data from middleware continuous integration systems -- **OpenShift CI**: Integration with OpenShift CI pipelines -- **S3 Storage**: Download benchmark data from S3-compatible storage -- **Prometheus**: Direct integration with Prometheus metrics - -### Analysis Methods -- **Hunter**: Advanced anomaly detection algorithm -- **Standard Deviation**: Statistical variance analysis -- **Z-Score**: Standardized score analysis -- **Custom Methods**: Extensible framework for additional analysis approaches - -## Installation - -### Prerequisites -- Python 3.7+ -- Access to Kubernetes/OpenShift cluster (for benchmark execution) -- Prometheus instance (for metrics collection) - -### Dependencies - -Install required Python packages: - -```bash -pip install -r requirements.txt -``` - -Key dependencies include: -- `pandas` - Data manipulation and analysis -- `dash` - Web-based interactive visualizations -- `plotly` - Plotting and charting -- `fire` - CLI framework -- `prometheus_api_client` - Prometheus integration -- `pydantic` - Data validation -- `opensearch-py` - OpenSearch integration -- `boto3` - AWS S3 integration - -## Usage - -Matrix Benchmarking provides a command-line interface with multiple subcommands: - -### Basic Commands - -```bash -# Run benchmarks -python -m matrix_benchmarking benchmark --workload --run - -# Download benchmark data -python -m matrix_benchmarking download --source - -# Parse benchmark results -python -m matrix_benchmarking parse --results-dir - -# Generate visualizations -python -m matrix_benchmarking visualize --results-dir - -# Analyze data with long-term storage -python -m matrix_benchmarking analyze_lts --method -``` - -### Long-term Storage Operations - -```bash -# Upload data to long-term storage -python -m matrix_benchmarking upload_lts --data-dir - -# Download from long-term storage -python -m matrix_benchmarking download_lts --output-dir - -# Generate storage schema -python -m matrix_benchmarking generate_lts_schema --config -``` - -### Getting Help - -```bash -# General help -python -m matrix_benchmarking --help - -# Command-specific help -python -m matrix_benchmarking benchmark --help -``` - -## Configuration - -Matrix Benchmarking uses YAML configuration files for: -- Benchmark definitions -- Data source configurations -- Analysis parameters -- Visualization settings - -Example configuration structure: -```yaml -workload: - name: "performance_test" - parameters: - # Workload-specific parameters - -data_sources: - prometheus: - url: "http://prometheus:9090" - -analysis: - methods: - - hunter - - stdev - - zscore -``` - -## Architecture - -### Module Structure - -- **`matrix_benchmarking/`** - Main package - - **`benchmark.py`** - Benchmark execution logic - - **`download.py`** - Data downloading orchestration - - **`parse.py`** - Result parsing utilities - - **`visualize.py`** - Visualization generation - - **`analyze_lts.py`** - Long-term storage analysis - - **`upload_lts.py`** - Long-term storage uploads - - **`download_lts.py`** - Long-term storage downloads - - **`generate_lts_schema.py`** - Schema generation - -### Submodules - -- **`downloading/`** - Data acquisition - - **`scrape/`** - Web scraping utilities - - `middleware_ci.py` - Middleware CI integration - - `ocp_ci.py` - OpenShift CI integration - - `s3.py` - S3 storage integration - -- **`exec/`** - Execution utilities - - `kube.py` - Kubernetes operations - - `prom.py` - Prometheus queries - - `common.py` - Shared execution logic - -- **`parsing/`** - Data parsing - - `prom.py` - Prometheus metrics parsing - -- **`plotting/`** - Visualization - - **`ui/`** - User interface components - - `web.py` - Web-based dashboards - - `report.py` - Report generation - -- **`store/`** - Data storage - - `simple.py` - Simple file-based storage - - `prom_db.py` - Prometheus database integration - -- **`analyze/`** - Analysis methods - - **`method/`** - Statistical analysis implementations - - `hunter.py` - Hunter anomaly detection - - `stdev.py` - Standard deviation analysis - - `zscore.py` - Z-score analysis - -## Web Interface - -Matrix Benchmarking includes a web-based interface for: -- Interactive data exploration -- Real-time visualization updates -- Report generation and sharing -- Performance trend analysis - -Access the web interface by running the visualize command with web output enabled. - -## Development - -### Project Structure -``` -matrix_benchmarking/ -├── matrix_benchmarking/ # Main package -│ ├── analyze/ # Analysis methods -│ ├── downloading/ # Data acquisition -│ ├── exec/ # Execution utilities -│ ├── parsing/ # Data parsing -│ ├── plotting/ # Visualization -│ └── store/ # Data storage -├── utils/ # Utility modules -├── requirements.txt # Python dependencies -└── README.md # This file -``` - -### Extending Analysis Methods - -To add a new analysis method: - -1. Create a new file in `matrix_benchmarking/analyze/method/` -2. Implement the analysis interface -3. Register the method in the analysis framework -4. Add tests and documentation - -### Adding Data Sources - -To support a new data source: - -1. Create a scraper in `matrix_benchmarking/downloading/scrape/` -2. Implement the downloading interface -3. Add configuration support -4. Update documentation - -## Contributing - -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Add tests for new functionality -5. Submit a pull request - -## License - -This project is part of the TOPSAIL project under the OpenShift PSAP organization. - -## Support - -For issues, questions, or contributions, please refer to the OpenShift PSAP team or create an issue in the repository. diff --git a/projects/matrix_benchmarking/subproject/bin/matbench b/projects/matrix_benchmarking/subproject/bin/matbench deleted file mode 100755 index c97cb5843..000000000 --- a/projects/matrix_benchmarking/subproject/bin/matbench +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -THIS_DIR=$(cd "$(dirname ${BASH_SOURCE[0]})">/dev/null 2>&1 && pwd) - -thisfile_realpath=$(realpath "${BASH_SOURCE[0]}") -thisfile_basedir=$(dirname "$thisfile_realpath")/.. -matbench_basedir=$(cd "${thisfile_basedir}" && pwd -P) - -export PYTHONPATH="${matbench_basedir}:${PYTHONPATH}" - -exec python3 -m matrix_benchmarking.main "$@" diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/__init__.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/__init__.py deleted file mode 100644 index 81371953d..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -import math -import types -from collections import defaultdict -from collections.abc import Callable -from typing import Optional - - -class RegressionStatus(types.SimpleNamespace): - def __init__( - self, - accepted: bool, - rating: int, - rating_color: str | None = None, - improved: int | None = None, - description: str | None = None, - details: dict | None = None, - details_fmt: dict | None = None, - details_conditional_fmt: Callable | None = None, - ): - self.rating = rating - self.rating_color = rating_color - self.improved = improved - self.description = description - self.accepted = accepted - - self.details = details - self.details_fmt = details_fmt - self.details_conditional_fmt = details_conditional_fmt - - -def do_regression_analyze(*args, **kwargs): - # lazy loading of hunter package ... - from .method import hunter as analyze_method - - return analyze_method.do_regression_analyze(*args, **kwargs) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/hunter.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/hunter.py deleted file mode 100644 index 596455205..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/hunter.py +++ /dev/null @@ -1,91 +0,0 @@ -from hunter import analysis -from matrix_benchmarking.analyze import RegressionStatus - -### - -WINDOW_LEN = 50 -MAX_PVALUE = 0.001 -MIN_MAGNITUDE = 0.05 - -### - - -def do_regression_analyze(current_value, historical_values, lower_better, kpi_unit): - values = historical_values + [current_value] - - change_points = analysis.compute_change_points( - values, window_len=WINDOW_LEN, max_pvalue=MAX_PVALUE, min_magnitude=MIN_MAGNITUDE - ) - - status = 0 - direction = 0 - explanation = "No change point detected with hunter" - details = [cp_to_details(p) for p in change_points] - - if len(change_points): - direction = change_points[-1].stats.forward_rel_change() - # direction > 0 --> value increased - # direction < 0 --> value decreated - decreased = direction < 0 - increased = not decreased - improved = (decreased and lower_better) or (increased and not lower_better) - - # if the current value is alone in its statistical group: - if change_points[-1].index == len(historical_values): - accepted = False - description = "Jump" if improved else "Regression" - rating = 1 - else: - accepted = True - description = f"Inline ({len(change_points)} historical)" - rating = 0 - - else: - accepted = True - description = "Inline" - rating = 0 - improved = None - - return RegressionStatus( - rating=rating, - accepted=accepted, - details=details, - improved=improved, - description=description, - # a function to format the content of the details cells - details_fmt=__get_details_fmt(kpi_unit), - # a function to style the details cells - details_conditional_fmt=__get_details_conditional_fmt, - ) - - -### - - -def __get_details_fmt(kpi_unit): - return dict() - - -def __get_details_conditional_fmt(row): - fmt = [] - for key, value in zip(row.keys(), row.values, strict=False): - style = "background: palegreen" - - fmt.append(style) - - return fmt - - -### - - -def cp_to_details(cp): - return { - "index": int(cp.index), - "mean_1": float(cp.stats.mean_1), - "mean_2": float(cp.stats.mean_2), - "std_1": float(cp.stats.std_1), - "std_2": float(cp.stats.std_2), - "pvalue": float(cp.stats.pvalue), - "forward_rel_change": cp.stats.forward_rel_change(), - } diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/skeleton.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/skeleton.py deleted file mode 100644 index 7b1791b14..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/skeleton.py +++ /dev/null @@ -1,68 +0,0 @@ -from matrix_benchmarking.analyze import RegressionStatus - -### - - -def do_regression_analyze(current_value, historical_values, lower_better, kpi_unit): - # anything useful to understand the analyze in the report - details = dict( - current_value=current_value, - historical_values=historical_values, - lower_better=lower_better, - kpi_unit=kpi_unit, - ) - - # indicator telling if the current value is better or worst than the historical values - improved = ( - current_value < min(historical_values) - if lower_better - else current_value > max(historical_values) - ) - - # indicator telling if the current value should be accepted or rejected (rejects will turn the CI red) - accepted = improved - - # a rating indicating the regression level of the current value against the historical values - # 0 is in-line, no deviation is measured - # <1 deviation is accepted - # >=1 deviation is too important - # 'improved' tells the direction of the deviation (better result or worst) - rating = 0.2 - - # human readable text describing the rating - description = "good" if improved else "bad" - - return RegressionStatus( - rating=rating, - accepted=accepted, - details=details, - improved=improved, - description=description, - # a function to format the content of the details cells - details_fmt=__get_details_fmt(kpi_unit), - # a function to style the details cells - details_conditional_fmt=__get_details_conditional_fmt, - ) - - -### - - -def __get_details_fmt(kpi_unit): - def historical_values_fmt(value): - return "[" + ", ".join([f"{v:.2f}" for v in value]) + "] " + kpi_unit - - return dict( - current_value="{:.2f} " + kpi_unit, - historical_values=historical_values_fmt, - ) - - -def __get_details_conditional_fmt(row): - fmt = [] - for key, value in zip(row.keys(), row.values, strict=False): - style = "background: palegreen" - - fmt.append(style) - - return fmt diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/stdev.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/stdev.py deleted file mode 100644 index bf43d3432..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/stdev.py +++ /dev/null @@ -1,223 +0,0 @@ -import statistics - -from matrix_benchmarking.analyze import RegressionStatus - -IMPROVED_DESCRIPTION = { - 0: "very close+", - 1: "in-line +", - 2: "improved", - 3: "improved +", - 4: "improved++", -} -DEGRADED_DESCRIPTION = { - 0: "very close-", - 1: "in-line -", - 2: "degraded", - 3: "degraded +", - 4: "degraded++", -} - -COLOR_STDEV_BOUND = "#32CD32" # Lime Green -COLOR_STDEV_NOT_BOUND = "#FF5733" # Red brick -COLOR_STDEV_NONE = "lightgreen" - -######## - -MAX_STDEV = 4 - -######## - - -def do_regression_analyze(current_value, historical_values, lower_better, kpi_unit): - details = dict( - current_value=current_value, - previous_mean=get_measure_of_mean(historical_values), - std_dev=get_measure_of_distribution(historical_values), - change=get_percentage_change(current_value, historical_values), - delta=get_delta(current_value, historical_values), - ) - - below = current_value < details["previous_mean"] - - improved = ( - (below and lower_better) or (not below and not lower_better) - if lower_better is not None - else None - ) - - found_in_stdev = MAX_STDEV - for deviation in range(1, MAX_STDEV): - dev_dist, dev_bound = get_std_dev_measurements(deviation, current_value, historical_values) - details[f"std_dev_{deviation}"] = dev_dist - if found_in_stdev != MAX_STDEV: - details[f"std_dev_{deviation}_bound"] = None - continue - - details[f"std_dev_{deviation}_bound"] = dev_bound - if dev_bound: - found_in_stdev = deviation - - description = (IMPROVED_DESCRIPTION if improved else DEGRADED_DESCRIPTION)[found_in_stdev] - rating = found_in_stdev / MAX_STDEV - accepted = improved or found_in_stdev < MAX_STDEV - - return RegressionStatus( - rating=rating, - accepted=accepted, - details=details, - details_fmt=__get_details_fmt(kpi_unit), - details_conditional_fmt=__get_details_conditional_fmt, - improved=improved, - description=description, - ) - - -###### -###### -###### - - -def __get_details_fmt(kpi_unit): - return dict( - current_value="{:.2f} " + kpi_unit, - previous_mean="{:.2f} " + kpi_unit, - std_dev="{:.1f}", - std_dev_1="{:.1f} %", - std_dev_2="{:.1f} %", - std_dev_3="{:.1f} %", - change="{:.1f} %", - delta="{:.1f}", - ) - - -def __get_details_conditional_fmt(row): - fmt = [] - for key, value in zip(row.keys(), row.values, strict=False): - style = "" - - if key in ["std_dev_1_bound", "std_dev_2_bound", "std_dev_3_bound"]: - if value is True: - style = f"background: {COLOR_STDEV_BOUND}" # green - elif value is False: - style = f"background: {COLOR_STDEV_NOT_BOUND}" # red - elif value is None: - style = f"background: {COLOR_STDEV_NONE}" - - fmt.append(style) - - return fmt - - -####################### -####################### -####################### - -# Calculate sample/population mean - - -def get_measure_of_mean(data): - """ - Returns Mean to determine the limits - """ - if len(data) == 1: - return data[-1] - - if len(data) == 0: - return 0 - - return statistics.mean(data) - - -# Calculate sample/population standard deviation - - -def get_measure_of_distribution(data): - """ - Returns STDDEV to determine the limits - """ - if len(data) == 1: - return data[-1] - - if len(data) == 0: - return 0 - - return statistics.stdev(data) - - -def get_std_dev_measurements(deviation, current_mean, previous_data): - """ - Returns standard deviation bounds - """ - - previous_data_mean = get_measure_of_mean(previous_data) - previous_data_stddev = get_measure_of_distribution(previous_data) - - if deviation == 1: - std_lower_bound = previous_data_mean - (previous_data_stddev * deviation) - std_upper_bound = previous_data_mean + (previous_data_stddev * deviation) - - number_of_observation = len( - [item for item in previous_data if item <= std_upper_bound and item >= std_lower_bound] - ) - if current_mean <= std_upper_bound and current_mean >= std_lower_bound: - bound_verify = True - else: - bound_verify = False - - else: - std_lower_bound_1 = previous_data_mean - (previous_data_stddev * (deviation - 1)) - std_upper_bound_1 = previous_data_mean + (previous_data_stddev * (deviation - 1)) - - std_lower_bound_2 = previous_data_mean - (previous_data_stddev * deviation) - std_upper_bound_2 = previous_data_mean + (previous_data_stddev * deviation) - - number_of_observation_below_the_mean = len( - [ - item - for item in previous_data - if item <= std_lower_bound_1 and item >= std_lower_bound_2 - ] - ) - number_of_observation_above_the_mean = len( - [ - item - for item in previous_data - if item >= std_upper_bound_1 and item <= std_upper_bound_2 - ] - ) - - number_of_observation = ( - number_of_observation_above_the_mean + number_of_observation_below_the_mean - ) - - if current_mean <= std_lower_bound_1 and current_mean >= std_lower_bound_2: - bound_verify = True - elif current_mean >= std_upper_bound_1 and current_mean <= std_upper_bound_2: - bound_verify = True - else: - bound_verify = False - - pct = 0 if len(previous_data) <= 1 else (number_of_observation / len(previous_data)) * 100 - - return pct, bound_verify - - -def get_percentage_change(mean, previous_data): - """ - Returns %change w.r.t the average mean - """ - - previous_data_mean = get_measure_of_mean(previous_data) - if previous_data_mean == 0: - return 100 - else: - return ((mean / previous_data_mean) * 100) - 100 - - -def get_delta(mean, previous_data): - """ - Returns the difference of current_mean and previous_mean - """ - - previous_data_mean = get_measure_of_mean(previous_data) - return mean - previous_data_mean diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/zscore.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/zscore.py deleted file mode 100644 index 717c2f267..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/method/zscore.py +++ /dev/null @@ -1,71 +0,0 @@ -import numpy as np -from matrix_benchmarking.analyze import RegressionStatus - -THRESHOLD = 3 - - -def do_regression_analyze(current_value, historical_values, lower_better, kpi_unit): - """ - Determine if the curr_result is more/less than threshold - standard deviations away from the previous_results - """ - - historical_mean = np.mean(historical_values) - if len(historical_values) == 1: - std = 0 - z_score = 0 - else: - std = np.std(historical_values) - z_score = (current_value - historical_mean) / std - - details = { - "current_value": current_value, - "historical_mean": historical_mean, - "threshold": THRESHOLD, - "zscore": z_score, - "stddev": std, - } - improved = True if z_score > 0 else False - if len(historical_values) == 1: - accepted = None - description = "Not enough historical values" - elif abs(z_score) < THRESHOLD: - accepted = True - description = "z-score lower than threshold" - else: - accepted = False - description = "z-score greater than threshold" - - rating = abs(z_score / THRESHOLD) - - if accepted is None: - rating = None - - return RegressionStatus( - rating=rating, - accepted=accepted, - details=details, - improved=improved, - description=description, - # a function to format the content of the details cells - details_fmt=__get_details_fmt(kpi_unit), - # a function to style the details cells - details_conditional_fmt=__get_details_conditional_fmt, - ) - - -### - - -def __get_details_fmt(kpi_unit): - return dict() - - -def __get_details_conditional_fmt(row): - fmt = [] - for key, value in zip(row.keys(), row.values, strict=False): - style = "background: palegreen" - - fmt.append(style) - - return fmt diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/report.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/report.py deleted file mode 100644 index 76e1af4b5..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze/report.py +++ /dev/null @@ -1,887 +0,0 @@ -import copy -import logging -import math -from collections import defaultdict - -import matrix_benchmarking.analyze as analyze -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.ui.report as plotting_ui_report -import pandas as pd -import plotly -import plotly.express as px -import plotly.io as pio -import yaml -from dash import html -from packaging.version import InvalidVersion, Version - -# if false, skip the entries without regression in the report -INCLUDE_ALL_THE_ENTRIES = True -# if false, skip the KPIs without regression in the report -INCLUDE_ALL_THE_KPIS = True -# if false, do not include the regression plots (~4.5MB per plot) -INCLUDE_REGRESSION_PLOT = False - -COLOR_OVERVIEW_NAME_META = "#F5F5DC" # beige -COLOR_OVERVIEW_NAME_VAR = "#ADD8E6" # light green - -COLOR_IMPROVED = "#32CD32" -COLOR_DEGRADED = "#FF5733" -COLOR_IMPROVED_NONE = "#ADD8E6" - -COLOR_NAN = "white; color: white" -COLOR_DEFAULT = "#CFFDBC" - - -def is_nan(entry): - return isinstance(entry, float) and math.isnan(entry) - - -def get_rating_color(rating, improved=True): - if is_nan(rating): - return COLOR_NAN - - OVERVIEW_IMPROVED = "#61c45a" - OVERVIEW_REGRESSED = "#ff7f00" - OVERVIEW_STABLE = "#e8fedd" - - if rating is None: - return OVERVIEW_STABLE - - if rating >= 1: - return OVERVIEW_IMPROVED if improved else OVERVIEW_REGRESSED - - # see also plotly.colors.sequential.Inferno_r and others - if improved: - color_scale = [OVERVIEW_STABLE, OVERVIEW_IMPROVED] - else: - color_scale = [OVERVIEW_STABLE, OVERVIEW_REGRESSED] - - color = plotly.colors.sample_colorscale(color_scale, samplepoints=[rating])[0] - - return color - - -COLOR_OVERVIEW_VARIABLE = "#ADD8E6" # Light Blue -COLOR_OVERVIEW_FIX = "#ccf0a2" # Light green - -# Declare properties to style the Table -STYLE_TABLE = [ - { - "selector": "tr:hover", - "props": [ - ("background-color", "yellow"), - ], - }, - { - "selector": "th", - "props": [ - ("background-color", "grey"), - ("border-collapse", "collapse"), - ], - }, - { - "selector": "td", - "props": [ - ("border-collapse", "collapse"), - ], - }, - { - "selector": "caption", - "props": [ - ("font-size", "1.875em"), - ("text-align", "center"), - ], - }, -] - -STYLE_HIDE_COLUMN_TITLES = [{"selector": "thead", "props": [("display", "none")]}] - - -class OvervallResult: - def __init__(self, rating, description, improved, current_value_str): - self.rating = rating - self.description = description - self.improved = improved - self.current_value_str = current_value_str - - -def longestCommonPrefix(strs): - if not all(isinstance(item, str) for item in strs): - return None - - if not strs or len(strs) == 1: - return "" - - min_s = min(strs) - max_s = max(strs) - - if not min_s: - return "" - for i in range(len(min_s)): - if max_s[i] != min_s[i]: - return max_s[:i] - return min_s[:] - - -def generate_regression_analyse_report( - regression_df, kpi_filter, comparison_keys, ignored_keys, sorting_keys -): - pio.renderers.default = "notebook" - - idx = 0 - total_points = 0 - warnings_already_shown = [] - failures = 0 - not_analyzed = 0 - no_history = 0 - significant_performance_increase = 0 - significant_performance_decrease = 0 - - report = [] - report.append(html.H1("Regression Analysis Summary")) - - summary_html = [] # will be populated after the loop - report.append(html.Div(summary_html)) - all_regr_results_data = [] - - all_lts_settings = defaultdict(set) - all_settings = defaultdict(set) - variables = [] - fix_settings = [] - lts_variables = [] - lts_fix_settings = [] - - for row_values in regression_df.values: - row = dict(zip(regression_df, row_values, strict=False)) - - for row_key, entry in zip(regression_df, row_values, strict=False): - if is_nan(entry): - continue - if row_key == "ref": - continue - - dest_all_settings = all_settings if row_key == row["ref"] else all_lts_settings - - entry_settings = ( - entry.results.lts.metadata.settings - if row_key == row["ref"] - else entry.results[0].results.metadata.settings - ) - - for k, v in entry_settings.__dict__.items(): - if isinstance(v, list): - v = str(v) - if not v.__hash__: - continue - if k in ignored_keys: - continue - dest_all_settings[k].add(v) - - for lts_meta_key in common.LTS_META_KEYS: - all_settings.pop(lts_meta_key, None) - all_lts_settings.pop(lts_meta_key, None) - - for k, v in all_settings.items(): - (variables if len(v) > 1 else fix_settings).append(k) - - variables.sort() - # take the sorting keys in the reversed order, - # so that sorting_keys[0] ends up in variables[0], etc - for key in sorting_keys[::-1]: - if key not in variables: - continue - # move the key first - variables.remove(key) - variables.insert(0, key) - - for k, v in all_lts_settings.items(): - (lts_variables if len(v) > 1 else lts_fix_settings).append(k) - - kpis_common_prefix = "" - for row_values in regression_df.values: - row = dict(zip(regression_df, row_values, strict=False)) - ref_entry = row[row["ref"]] - kpis = ref_entry.results.lts.kpis - - kpis_common_prefix = longestCommonPrefix(list(kpis.keys())) - break - - for row_values in regression_df.values: - idx += 1 - - logging.info(f"Processing entry # {idx} ...") - - row = dict(zip(regression_df, row_values, strict=False)) - - ref_entry = row[row["ref"]] - kpis = ref_entry.results.lts.kpis - - metadata = copy.copy(ref_entry.results.lts.metadata) - metadata_settings = copy.copy(metadata.__dict__.pop("settings")) - - regression_name = "|".join(f"{k}={metadata_settings.__dict__[k]}" for k in variables) - if not regression_name: - regression_name = "Unique test" - - entry_report = [] - entry_report.append(html.H1(["Entry #", idx, " – ", regression_name])) - - # entry header - - has_history = True - if len([... for c in row.values() if not is_nan(c)]) == 2: - entry_report.append(html.P(html.B("No historical records ..."))) - no_history += 1 - has_history = False - - entry_report += _generate_entry_header(metadata_settings, metadata) - - entry_regr_results = dict(entry_id=idx) - metadata_column_count = len(entry_regr_results) - - for var in variables: - entry_regr_results[var] = metadata_settings.__dict__[var] - if not variables: - entry_regr_results["name"] = regression_name - - include_this_entry_in_report = False - - for kpi in kpis: - if kpi_filter and kpi_filter not in kpi: - continue - - ref_kpi = ref_entry.results.lts.kpis[kpi] - if isinstance(ref_kpi.value, list): - continue - - if ref_kpi.ignored_for_regression: - continue - - ref_name = row["ref"] - - comparison_data = [] - - ref_line = dict() - for k in comparison_keys: - ref_line[k] = ref_kpi.__dict__.get(k) - - # keep this vvv below ^^^ to preserve the order in the rendered table - ref_line["value"] = ref_kpi.value - ref_line["ref"] = "*" - - if formatted_value := format_kpi_value(ref_kpi, want_raw_value=False): - ref_line["formatted value"] = formatted_value - - comparison_data.append(ref_line) - - historical_values = [] - - for comparison_key_value in sorted(row.keys()): - if comparison_key_value == "ref": - continue - - comparison_gathered_entries = row[comparison_key_value] - if is_nan(comparison_gathered_entries): - continue - if comparison_key_value == row["ref"]: - continue - - entry = comparison_gathered_entries.results[0] - - entry_name = { - k: entry.results.metadata.settings.__dict__[k] for k in comparison_keys - } - - records_count = len(comparison_gathered_entries.results) - if records_count != 1: - msg = f"Multiple records ({records_count}) found for {entry_name}. Taking the first one." - if msg not in warnings_already_shown: - warnings_already_shown.append(msg) - logging.warning(msg) - - kpi_entry = entry.results.kpis.__dict__.get(kpi, None) - # ignore the the KPI isn't part of the current entry - if kpi_entry is None: - continue - - history_line = dict( - ref="", - value=kpi_entry.value, - ) - - if formatted_value := format_kpi_value( - kpi_entry, want_raw_value=False, fmt_kpi=ref_kpi - ): - history_line["formatted value"] = formatted_value - - comparison_data.append(history_line | entry_name) - historical_values.append(kpi_entry.value) - - # Comparison table - - lower_better = getattr(ref_kpi, "lower_better", None) - - if lower_better is None: - msg = f"KPI '{kpi}' does not define the 'lower_better' property :/ " - if msg not in warnings_already_shown: - logging.warning(msg) - warnings_already_shown.append(msg) - - regr_result = analyze.do_regression_analyze( - ref_kpi.value, historical_values, lower_better=lower_better, kpi_unit=ref_kpi.unit - ) - - validate_regression_result(regr_result) - - current_value_str = format_kpi_value(ref_kpi) - - entry_regr_results[kpi.replace(kpis_common_prefix, "")] = OvervallResult( - regr_result.rating, - regr_result.description, - regr_result.improved, - current_value_str=current_value_str, - ) - - include_this_kpi_in_report = False - if has_history: - total_points += 1 - - if regr_result.improved and regr_result.rating >= 1: - include_this_kpi_in_report = True - failures += 1 - significant_performance_increase += 1 - elif regr_result.accepted is False: - include_this_kpi_in_report = True - failures += 1 - significant_performance_decrease += 1 - elif regr_result.accepted is None: - not_analyzed += 1 - - if include_this_kpi_in_report: - include_this_entry_in_report = True - - if INCLUDE_ALL_THE_ENTRIES: - include_this_entry_in_report = True - include_this_kpi_in_report = True - - if not include_this_kpi_in_report: - continue - - comparison_df = _generate_sorted_pd_table(comparison_data, comparison_keys).sort_values( - by="ref" - ) - - # KPI header - - comparison_df = comparison_df.sort_values(comparison_keys).reset_index(drop=True) - entry_report.append(html.H2(f"Entry #{idx} KPI {kpi}")) - entry_report.append(html.P(html.I(f"{ref_kpi.help} (in {ref_kpi.unit})."))) - entry_report.append(_generate_comparison_table(comparison_df, ref_kpi.unit)) - - if not has_history: - entry_report.append(html.P(html.B("No historical records ..."))) - continue - - # Evaluation results - - entry_report.append(html.H3("Evaluation results")) - entry_report.append(_generate_evaluation_results(regr_result)) - - # Details - - entry_report.append(html.H3("Details")) - entry_report.append(_generate_details_table(regr_result)) - - # Plot - if INCLUDE_REGRESSION_PLOT: - entry_report.append( - _generate_comparison_plot( - comparison_df, comparison_keys, kpi, ref_kpi, kpis_common_prefix - ) - ) - - if not include_this_entry_in_report: - continue - - report += entry_report - all_regr_results_data.append(entry_regr_results) - - # Configuration overview - - summary_html.append(html.H2("Configuration overview")) - summary_html.append(_generate_configuration_overview(all_settings, variables)) - - # LTS overview - summary_html.append(html.H2("Historical records overview")) - - if no_history > 0: - summary_html.append( - html.P( - f"No historical records for {no_history} entrie{'s' if no_history > 1 else ''} ..." - ) - ) - - if all_lts_settings: - summary_html.append(_generate_lts_overview(all_lts_settings, lts_variables)) - - # Results overview - - summary_html.append(html.H2("Results overview")) - kpi_names = ( - set( - list(all_regr_results_data[0].keys())[ - max([metadata_column_count, len(variables) + 1]) : - ] - ) - if all_regr_results_data - else set() - ) - - summary_html.append( - _generate_results_overview(all_regr_results_data, variables, kpi_names, kpis_common_prefix) - ) - - # Summary - - summary = f"Performed {total_points} KPI regression analyses over {idx} entries x {len(kpi_names)} KPIs. {failures} KPIs didn't pass." - if no_history: - summary += f" {no_history} entries didn't have historical records." - if not_analyzed: - summary += ( - f" {not_analyzed} entries couldn't be analyzed (check the logs for an explanation). " - ) - if significant_performance_decrease: - summary += ( - f" {significant_performance_increase} KPIs had a significant performance decrease. " - ) - if significant_performance_increase: - summary += ( - f" {significant_performance_increase} KPIs had a significant performance increase. " - ) - - logging.info(summary) - summary_html.append(html.P(html.B(summary))) - - yaml_summary = dict( - total_points=total_points, - entries_count=idx, - kpis_count=len(kpi_names), - failures=failures, - no_history=no_history, - not_analyzed=not_analyzed, - significant_performance_increase=significant_performance_increase, - significant_performance_decrease=significant_performance_decrease, - message=summary, - ) - - return html.Span(report), failures, yaml_summary - - -def generate_and_save_regression_analyse_report( - report_dest, - yaml_dest, - regression_df, - kpi_filter, - comparison_key, - ignored_keys, - sorting_keys, -): - report, failures, yaml_summary = generate_regression_analyse_report( - regression_df, kpi_filter, comparison_key, ignored_keys, sorting_keys - ) - - if yaml_dest: - logging.info(f"Saving the YAML summary into {yaml_dest} ...") - with open(yaml_dest, "w") as f: - yaml.dump(yaml_summary, f) - # print("", file=f) - - if report_dest is None: - logging.warning("Skipping report generation.") - return failures - - logging.info(f"Saving the HTML report into {report_dest} ...") - plotting_ui_report.generate(None, report_dest, report, None, include_header=False) - - return failures - - -def _generate_entry_header(metadata_settings, metadata): - header = [] - - # --- URLs --- - - if hasattr(metadata_settings, "urls") or hasattr(metadata, "urls"): - urls = metadata_settings.urls if hasattr(metadata_settings, "urls") else metadata.urls - html_urls = [] - for url_name, url in urls.items(): - html_urls.append(html.Li(html.A(url_name, href=url))) - header.append(html.Ul(html_urls)) - - # --- LTS Metadata --- - - header.append(html.H3("Settings")) - - metadata.__dict__.pop("urls", None) - metadata.__dict__.pop("test_path", None) - metadata.__dict__.pop("run_id", None) - metadata.__dict__.pop("config", None) - - lts_metadata_df = pd.DataFrame(metadata.__dict__.items()) - lts_metadata_html = lts_metadata_df.style.hide(axis="index").set_table_styles( - table_styles=STYLE_TABLE + STYLE_HIDE_COLUMN_TITLES - ) - - header.append(lts_metadata_html) - - # --- KPI Labels/Settings --- - - header.append(html.H3("Labels")) - - metadata_settings.__dict__.pop("urls", None) - metadata_settings.__dict__.pop("test_path", None) - metadata_settings.__dict__.pop("run_id", None) - - kpi_settings_df = pd.DataFrame(metadata_settings.__dict__.items()) - - kpi_settings_html = kpi_settings_df.style.hide(axis="index").set_table_styles( - table_styles=STYLE_TABLE + STYLE_HIDE_COLUMN_TITLES - ) - - header.append(kpi_settings_html) - - return header - - -def _generate_sorted_pd_table(comparison_data, comparison_keys, warn=True): - comparison_df = pd.DataFrame(comparison_data) - - common_prefixes = {} - - for comparison_key in comparison_keys: - common_prefixes[comparison_key] = longestCommonPrefix( - list(comparison_df[comparison_key].values) - ) - - def to_version(value): - try: - return Version(str(value)) - except InvalidVersion: - pass - safe_value = "1+" + str(value).replace("/", ".").replace(":", ".").replace(", ", "-") - try: - return Version(safe_value) - except InvalidVersion: - if warn: - logging.warning(f"Cannot parse '{value}' as a version :/ ({safe_value})") - return Version("1.0") # ordering will be wrong - - def create_sort_index(_row): - sort_index = [] - - for comparison_key in comparison_keys: - common_prefix = common_prefixes[comparison_key] - value = _row[comparison_key] - if common_prefix: - value = value.removeprefix(common_prefix) - - sort_index += [to_version(value)] - - return sort_index - - comparison_df["__sort_index"] = comparison_df.apply(create_sort_index, axis=1) - - return comparison_df.sort_values("__sort_index").drop("__sort_index", axis=1) - - -def _generate_comparison_table(comparison_df, kpi_unit): - comparison_format = dict( - value="{:.2f} " + kpi_unit, - ) - - comparison_df_html = comparison_df.style.format(comparison_format).set_table_styles( - table_styles=STYLE_TABLE - ) - - return comparison_df_html - - -def _generate_evaluation_results(regr_result): - def rating_fmt(value): - if value is None: - return "---" - - return f"{value * 100:.0f}%" - - regr_results_evaluation_fmt = dict(rating=rating_fmt) - - def regr_results_evaluation_style(row): - fmt = [] - for key, value in zip(row.keys(), row.values, strict=False): - if key in ("improved", "accepted"): - color = ( - (COLOR_IMPROVED if value else COLOR_DEGRADED) - if value is not None - else COLOR_IMPROVED_NONE - ) - - if key in ("rating", "description"): - color = row.rating_color - - style = f"background: {color}" - fmt.append(style) - - return fmt - - rating_color = get_rating_color(regr_result.rating, regr_result.improved) - regr_df_html = ( - pd.DataFrame( - [ - dict( - improved=regr_result.improved, - rating=regr_result.rating, - rating_color=rating_color, - description=regr_result.description, - accepted=regr_result.accepted, - ) - ] - ) - .style.format(regr_results_evaluation_fmt) - .apply(regr_results_evaluation_style, axis=1) - .hide(["rating_color"], axis=1) - .hide(axis="index") - .set_table_styles(table_styles=STYLE_TABLE) - ) - - return regr_df_html - - -def _generate_lts_overview(all_lts_settings, lts_variables): - lts_config_overview_data = [] - for k, v in all_lts_settings.items(): - val = ", ".join(map(str, sorted(v))) - lts_config_overview_data.append(dict(name=k, value=val)) - - def lts_config_overview_conditional_format(row): - color = COLOR_OVERVIEW_VARIABLE if row["name"] in lts_variables else COLOR_OVERVIEW_FIX - - return [f"background: {color}"] * len(row) - - lts_config_overview_df = pd.DataFrame(lts_config_overview_data) - lts_config_overview_df_html = ( - lts_config_overview_df.style.apply(lts_config_overview_conditional_format, axis=1) - .hide(axis="index") - .set_table_styles(table_styles=STYLE_TABLE) - ) - - return lts_config_overview_df_html - - -def _generate_configuration_overview(all_settings, variables): - config_overview_data = [] - - for k, v in all_settings.items(): - try: - sorted_values = sorted(v) - except TypeError: - # eg: '<' not supported between instances of 'NoneType' and 'bool' - sorted_values = sorted(map(str, v)) - - val = ", ".join(map(str, sorted_values)) - config_overview_data.append(dict(name=k, value=val)) - - def config_overview_conditional_format(row): - color = COLOR_OVERVIEW_VARIABLE if row["name"] in variables else COLOR_OVERVIEW_FIX - - return [f"background: {color}"] * len(row) - - config_overview_df = pd.DataFrame(config_overview_data) - config_overview_df_html = ( - config_overview_df.style.apply(config_overview_conditional_format, axis=1) - .hide(axis="index") - .set_table_styles(table_styles=STYLE_TABLE) - ) - - return config_overview_df_html - - -def _generate_results_overview( - all_regr_results_data, variables, kpi_names, kpis_common_prefix, warn=True -): - all_regr_results_df = _generate_sorted_pd_table(all_regr_results_data, variables, warn=warn) - - def fmt(value): - if is_nan(value): - return "" - - if isinstance(value, int) or isinstance(value, str): - return value - - return value.current_value_str - - overview_fmt = dict.fromkeys(kpi_names, fmt) - - kpis_to_hide = [] - if not INCLUDE_ALL_THE_KPIS: - for kpi_name in kpi_names: - has_regression = False - for kpi_value in all_regr_results_df[kpi_name].values: - if kpi_value.rating != 0: - has_regression = True - break - if not has_regression: - kpis_to_hide.append(kpi_name) - - all_regr_results_df_html = ( - all_regr_results_df.style.apply(get_all_regr_results_conditional_format(variables), axis=1) - .format(overview_fmt) - .hide(axis="index") - .hide(kpis_to_hide, axis=1) - .set_table_styles(table_styles=STYLE_TABLE) - ) - - return all_regr_results_df_html - - -def get_all_regr_results_conditional_format(variables): - def all_regr_results_conditional_format(row): - fmt = [] - - for key, value in zip(row.keys(), row.values, strict=False): - if isinstance(value, OvervallResult): - color = get_rating_color(value.rating, value.improved) - style = f"background: {color}; text-align: right;" - elif key in variables: - style = f"background: {COLOR_OVERVIEW_NAME_VAR}; text-align: center;" - elif is_nan(value): - style = "background: red" - else: - style = f"background: {COLOR_OVERVIEW_NAME_META}; text-align: center;" - - fmt.append(style) - - return fmt - - return all_regr_results_conditional_format - - -def validate_regression_result(regr_result): - if regr_result is None: - raise ValueError("The regr_result is None ...") - - if regr_result.accepted is None: - return - - if "inf" in str(regr_result.rating): - raise ValueError("The regr_result rating shouldn't be infinite ...") - - if regr_result.rating < 0: - raise ValueError( - f"The regr_result rating ({regr_result.rating:.2f}) should not be negative" - ) - - range_0_1 = 0 <= regr_result.rating < 1 - if not ((regr_result.accepted and range_0_1) or (not regr_result.accepted and not range_0_1)): - if not regr_result.improved: - raise ValueError( - f"The regr_result rating ({regr_result.rating:.2f}) should be between 0 and 1 only when the value is accepted ({regr_result.accepted})..." - ) - - -def _generate_details_table(regr_result): - details_data = ( - [regr_result.details] if isinstance(regr_result.details, dict) else regr_result.details - ) - regr_df_html = ( - pd.DataFrame(details_data) - .style.apply(regr_result.details_conditional_fmt, axis=1) - .format(regr_result.details_fmt) - .hide(axis="index") - .set_table_styles(table_styles=STYLE_TABLE) - ) - - return regr_df_html - - -def _generate_comparison_plot( - _comparison_df, comparison_keys, kpi_name, ref_kpi, kpis_common_prefix -): - data = [] - ref = None - - def create_name_column(row): - return "|".join(f"{k}={row[k]}" for k in comparison_keys) - - comparison_df = _comparison_df.copy() - comparison_df["name"] = comparison_df.apply(create_name_column, axis=1) - - fig = px.line( - comparison_df, - x="name", - y="value", - markers=True, - title=f"{ref_kpi.help} (in {ref_kpi.unit})", - ) - - ref_name = comparison_df[comparison_df.ref == "*"].name.iloc[0] - fig.add_hline(y=ref_kpi.value) - fig.add_vline(x=ref_name) - - y_name = f"{kpi_name.replace(kpis_common_prefix, '')} (in {ref_kpi.unit})" - if ref_kpi.lower_better is True: - y_name = f"❮ {y_name}. Lower is better." - elif ref_kpi.lower_better is False: - y_name = f"{y_name}. Higher is better. ❯ " - # ref_kpi.lower_better is None: nothing - - x_name = "|".join(comparison_keys) - fig.update_yaxes(title=y_name, range=[0, comparison_df.value.max() * 1.1]) - fig.update_xaxes(title=x_name) - - title = f"{kpi_name}
{ref_kpi.help} (in {ref_kpi.unit})" - - fig.update_layout( - title=title, - title_x=0.5, - ) - - # not using dcc.Graph() here, so this will follow another path than plots in plotting reports. - # here, fig.to_html() will be called. - return html.Div([fig], style=dict(height="525px", width="100%")) - - -def format_kpi_value(kpi, want_raw_value=True, fmt_kpi=None): - if fmt_kpi is None: - fmt_kpi = kpi - - kpi_is_list = isinstance(kpi.value, list) - - if not want_raw_value and not fmt_kpi.divisor: - return None - - if kpi.value is None: - return f"--- {kpi.unit}" - - elif fmt_kpi.full_format: - return kpi.full_format(kpi) - - elif fmt_kpi.format: - kpi_value = kpi.value - if fmt_kpi.divisor: - if kpi_is_list: - kpi_value = [(v / fmt_kpi.divisor) for v in kpi_value] - else: - kpi_value = kpi.value / fmt_kpi.divisor - - if kpi_is_list: - return ( - ", ".join([fmt_kpi.format.format(v) for v in kpi_value]) - + f" {fmt_kpi.divisor_unit}" - ) - else: - return fmt_kpi.format.format(kpi_value) + f" {fmt_kpi.divisor_unit}" - else: - if kpi_is_list: - return ", ".join([f"{v:.0f}" for v in kpi.value]) + f" {fmt_kpi.divisor_unit}" - else: - return f"{kpi.value:.0f} {kpi.unit}" diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py deleted file mode 100644 index 9005525ea..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/analyze_lts.py +++ /dev/null @@ -1,127 +0,0 @@ -import importlib -import logging -import pathlib -import sys - -import matrix_benchmarking.analyze.report as analyze_report -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.store as store - -LTS_ANCHOR_NAME = "source.lts.yaml" - - -def main( - workload: str = "", - workload_base_dir: str = "", - results_dirname: str = "", - lts_results_dirname: str = "", - filters: list[str] = [], - report_dest: str = "regression_report.html", - summary_yaml_dest: str = "regression_summary.yaml", - kpi_filter: str = "", -): - """ - Analyze MatrixBenchmark LTS results - - Analyze MatrixBenchmarking LTS results to detect performance regression - - Env: - MATBENCH_WORKLOAD - MATBENCH_WORKLOAD_BASE_DIR - MATBENCH_RESULTS_DIRNAME - MATBENCH_LTS_RESULTS_DIRNAME - MATBENCH_FILTERS - MATBENCH_REPORT_DEST - Args: - workload: Name of the workload to execute. (Mandatory.) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - results_dirname: Name of the directory where the results are stored. (Mandatory.) - lts_results_dirname: Name of the directory where the LTS results are stored. (Mandatory.) - filters: If provided, analyze only the experiment matching the filters. Eg: expe=expe1:expe2,something=true. - report_dest: Where to save the regression analyses report - kpi_filter: Filter (substring) that must be part of the KPI name to include it in the regression analyses - summary_yaml_dest: Where to save the YAML summary of the regression analyses - """ - - kwargs = dict(locals()) # capture the function arguments - - cli_args.setup_env_and_kwargs(kwargs) - cli_args.check_mandatory_kwargs(kwargs, ("workload", "results_dirname", "lts_results_dirname")) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="analyze-lts") - store_dir = pathlib.Path(kwargs["report_dest"]).parent - store_dir.mkdir(parents=True, exist_ok=True) - - workload_store = store.load_workload_store(kwargs) - - logging.info("Loading results ... ") - - workload_store.parse_data() - common.Matrix.uniformize_settings_keys() - common.Matrix.print_settings_to_log() - - if not common.Matrix.processed_map: - logging.error("Not result found.") - (store_dir / "NO_RESULTS_FOUND").touch() - return 1 - - logging.info(f"Loading results ... done. Found {common.Matrix.count_records()} results.") - logging.info("") - logging.info("--- LTS --- ") - - workload_store.parse_lts_data() - - common.LTS_Matrix.print_settings_to_log() - - logging.info( - f"Loading LTS results ... done. Found {common.LTS_Matrix.count_records()} results." - ) - if common.LTS_Matrix.count_records() == 0: - logging.error("Not LTS result found, exiting.") - (store_dir / "NO_HISTORICAL_RESULTS").touch() - - workload_analyze = get_workload_analyze_module(workload_store) - - regression_df, comparison_keys, ignored_keys, sorting_keys = workload_analyze.prepare() - - try: - failures = analyze_report.generate_and_save_regression_analyse_report( - kwargs["report_dest"], - kwargs["summary_yaml_dest"], - regression_df, - kwargs["kpi_filter"], - comparison_keys, - ignored_keys, - sorting_keys, - ) - - logging.info(f"The regression analyze finished with code {failures}.") - except Exception as e: - with open(store_dir / "FAILURE.txt", "a") as f: - print(str(e), file=f) - import traceback - - print(traceback.format_exc(), file=f) - raise e - - return failures - - return cli_args.TaskRunner(run) - - -def get_workload_analyze_module(workload_store): - workload_package = workload_store.__package__.removesuffix(".store") - module = f"{workload_package}.analyze" - logging.info(f"Loading {module} module ...") - - try: - analyze_module = importlib.import_module(module) - except ModuleNotFoundError as e: - logging.critical(f"Module {module} does not exist :/ ({e})") - sys.exit(1) - - logging.info(f"Loading {module} module ... done.") - - return analyze_module diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/benchmark.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/benchmark.py deleted file mode 100644 index 065f22959..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/benchmark.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -import sys - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.matrix as matrix -import matrix_benchmarking.store as store - - -# default values must evaluate to False, otherwise they cannot be -# overriden in the benchmark file. -def main( - workload: str = "", - workload_base_dir: str = "", - benchmark_file: str = "", - results_dirname: str = "", - run: bool = False, - generate: bool = False, - remote_mode: bool = False, - path_tpl: str = None, - script_tpl: str = None, - stop_on_error: bool = False, - expe_to_run: list[str] = [], - filters: list[str] = [], -): - """ - Run MatrixBenchmarking benchmarking. - - Run MatrixBenchmarking benchmarking. - - Env: - MATBENCH_WORKLOAD - MATBENCH_WORKLOAD_BASE_DIR - MATBENCH_BENCHMARK_FILE - MATBENCH_RUN - MATBENCH_RESULTS_DIRNAME - MATBENCH_REMOTE_MODE - MATBENCH_PATH_TPL - MATBENCH_SCRIPT_TPL - MATBENCH_STOP_ON_ERROR - MATBENCH_EXPE_TO_RUN - MATBENCH_FILTERS - - See the `FLAGS` section for the descriptions. - - Args: - workload: Name of the workload to execute. (Mandatory.) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - benchmark_file: Path of the benchmark file to execute. (Mandatory.) - - run: If 'False', run in dry mode. If 'True', execute the benchmark. - results_dirname: Name of the directory where the results will be stored. Can be set in the benchmark file. (Mandatory.) - remote_mode: If 'True', generate a benchmark script, instead of running the benchmark locally. If 'False', run the benchmark locally (default). Can be set in the benchmark file. - path_tpl: Path template for generating the directories where the benchmark results will be stored. Can be set in the benchmark file. - script_tpl: Path template for the script to execute the benchmark. Can be set in the benchmark file. - stop_on_error: If 'True', stop the matrix benchmarking execution on the first error. If 'False', ignore the error and continue. Can be set in the benchmark file. - expe_to_run: Experiments to run. Can be set in the benchmark file. - filters: If provided, parse only the experiment matching the filters. Eg: expe=expe1:expe2,something=true. - - """ - kwargs = dict(locals()) # capture the function arguments - - cli_args.setup_env_and_kwargs(kwargs) - - benchmark_file = kwargs["benchmark_file"] - try: - benchmark_yaml_file = cli_args.get_benchmark_yaml_file(benchmark_file) - except (ValueError, FileNotFoundError) as e: - logging.error(f"Failed to parse the benchmark file: {e}") - raise SystemExit(1) - - cli_args.update_kwargs_with_benchmark_file(kwargs, benchmark_yaml_file) - - cli_args.check_mandatory_kwargs(kwargs, ("workload", "results_dirname", "expe_to_run")) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="benchmark") - - # --- - - workload_store = store.load_workload_store(kwargs) - - # --- - - dry = not run - - logging.info("Loading previous results ... ") - workload_store.parse_data() - logging.info( - f"Loading previous results: done, found {len(common.Matrix.processed_map)} results" - ) - common.Matrix.uniformize_settings_keys() - - if dry: - logging.info("#") - logging.info("# DRY RUN") - logging.info("#") - - failed_count = matrix.Matrix(benchmark_yaml_file).run() - - sys.exit(failed_count) - - return cli_args.TaskRunner(run) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/cli_args.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/cli_args.py deleted file mode 100644 index 0afd29e27..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/cli_args.py +++ /dev/null @@ -1,208 +0,0 @@ -import json -import logging -import os -import pathlib -import sys - -import yaml - -kwargs = None -experiment_filters = {} -cli_environ = {} - - -def store_kwargs(_kwargs, *, execution_mode): - global kwargs - kwargs = _kwargs - kwargs["execution_mode"] = execution_mode - - -def get_benchmark_yaml_file(benchmark_file): - if not benchmark_file: - benchmark_file = os.environ.get("MATBENCH_BENCHMARK_FILE") - - if not benchmark_file: - raise ValueError( - "benchmark_file must be passed in the command-line or with the MATBENCH_BENCHMARK_FILE environment variable." - ) - - benchmark_file_path = pathlib.Path(os.path.realpath(benchmark_file)) - - if not benchmark_file_path.exists(): - raise FileNotFoundError(f"'{benchmark_file}' does not exit.") - if not benchmark_file_path.is_file(): - raise FileNotFoundError(f"'{benchmark_file}' must be a file.") - - # parse the benchmark file YAML - logging.info(f"Loading the benchmark file {benchmark_file} ...") - with open(benchmark_file_path) as f: - benchmark_yaml_file = yaml.safe_load(f) - - return benchmark_yaml_file - - -def update_env_with_env_files(): - """ - Overrides the function default args with the flags found in the environment variables files - """ - - for env in ( - ".env", - ".env.generated", - ".env.yaml", - ".env.json", - ".env.generated.json", - ".env.generated.yaml", - ): - env_file = pathlib.Path(env) - if not env_file.exists(): - continue - with open(env_file) as f: - if env_file.suffix in (".yaml", ".json"): - try: - doc = yaml.safe_load(f) if env_file.suffix == ".yaml" else json.load(f) - except Exception as e: - logging.error(f"Env file {env_file} is not a valid json/yaml file: {e}") - continue - - if not doc: - logging.warning(f"Env file {env_file} is empty") - continue - - if not hasattr(doc, "items"): - logging.warning(f"Env file {env_file} isn't a dictionnary") - continue - - for k, v in doc.items(): - if not isinstance(k, str): - logging.warning( - f"Env file {env_file} contains a key {k} that isn't a string ({k.__class__.__name__})" - ) - continue - - key = f"MATBENCH_{k.upper()}" - cli_environ[key] = str(v) - else: - for line in f.readlines(): - key, found, value = line.strip().partition("=") - if not found: - logging.warning(f"invalid line in {env}: {line.strip()}") - continue - if key in os.environ and os.environ[key]: - continue # prefer env to env file - cli_environ[key] = value - - -def update_kwargs_with_env(kwargs): - # override the function default args with the flags found in the environment variables - - for flag, current_value in kwargs.items(): - if current_value: - continue # already set, ignore. - key = f"MATBENCH_{flag.upper()}" - env_value = os.environ.get(key) - if not env_value: - # not set as environment var - env_value = cli_environ.get(key) - if not env_value: - continue # not set in an environment file, ignore - - kwargs[flag] = env_value # override the function arg with the environment variable value - - -def update_kwargs_with_benchmark_file(kwargs, benchmark_desc_file): - # override the function default args with the flags found in the benchmark file - for flag, current_value in kwargs.items(): - if current_value: - continue # already set, ignore. - - for item_name in (f"--{flag.replace('_', '-')}", f"--{flag}"): - file_value = benchmark_desc_file.get(item_name) - if file_value is None: - continue # not set, ignore. - kwargs[flag] = file_value # override the function arg with the benchmark file value - - del benchmark_desc_file[item_name] - break - - # warn for every flag remaining in the benchmark file - for key, value in benchmark_desc_file.items(): - if not key.startswith("--"): - continue - - logging.warning(f"unexpected flag found in the benchmark file: {key} = '{value}'") - - -def check_mandatory_kwargs(kwargs, mandatory_flags, sensitive_flags=[]): - command = [f"matbench {sys.argv[1]}"] - for k, v in sorted(kwargs.items()): - if v not in mandatory_flags and not v: - continue - - value = "" if k in sensitive_flags else v - command += [f"--{k.replace('_', '-')}='{value}'"] - - logging.info("MatrixBenchmarking starting with:\n" + " \\\n\t".join(command)) - - err = False - for flag in mandatory_flags: - if kwargs.get(flag): - continue - - logging.fatal( - f"--{flag} must be set in CLI, in the benchmark file or though MATBENCH_{flag.upper()} environment variable." - ) - err = True - - if err: - raise SystemExit(1) - - -def setup_env_and_kwargs(kwargs): - # overriding order: env file <- env var <- benchmark file <- cli - update_env_with_env_files() - update_kwargs_with_env(kwargs) - - filters = kwargs.get("filters") - if isinstance(filters, bool): - filters = str(filters) - - if filters: - parse_filters(filters) - - -def parse_filters(filters): - for kv in filters.split(","): - key, found, value = kv.partition("=") - if not found: - logging.error(f"Unexpected filter value: {kv}") - sys.exit(1) - - value = value.replace("\\:", "") - value = value.split(":") if ":" in value else [value] - value = [v.replace("", ":") for v in value] - - experiment_filters[key] = value - - -class TaskRunner: - """ - TaskRunner - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - If you're seeing this text, put the --help flag earlier in your list - of command-line arguments, this is a limitation of the CLI parsing library - used by the MatrixBenchmarking. - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - """ - - def __init__(self, run): - self.run = run - - def __str__(self): - return "---" diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/common.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/common.py deleted file mode 100644 index 53ca9df47..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/common.py +++ /dev/null @@ -1,245 +0,0 @@ -import itertools -import logging -import types -from collections import defaultdict -from collections.abc import Iterator - -MISSING_SETTING_VALUE = None - - -class MatrixEntry(types.SimpleNamespace): - def __init__( - self, - location, - results, - exit_code, - processed_key, - import_key, - processed_settings, - import_settings, - matrix, - settings=None, - stats=None, - is_gathered=None, - ): - self.is_gathered = False - - self.settings = settings or types.SimpleNamespace() - - self.stats = {} - - self.location = location - self.results = results - self.exit_code = exit_code - - self.settings.__dict__.update(processed_settings) - - self.processed_key = processed_key - self.import_settings = processed_settings - - matrix.import_map[import_key] = matrix.processed_map[processed_key] = self - - keys_to_skip = set() - for k, v in processed_settings.items(): - if not k.__hash__: - # logging.warn(f"Key {k} not unhashable (type {k.__class__}). Skipping it.") - keys_to_skip.add(k) - continue - elif not v.__hash__: - # logging.warn(f"Value {k}={v} not unhashable (type {k.__class__}). Skipping this key.") - keys_to_skip.add(k) - - [matrix.settings[k].add(v) for k, v in processed_settings.items() if k not in keys_to_skip] - - def get_name(self, variables) -> str: - return ", ".join( - [ - f"{key}={self.settings.__dict__[key]}" - for key in variables - if self.settings.__dict__[key] is not MISSING_SETTING_VALUE - and len([v for v in Matrix.settings[key] if v is not MISSING_SETTING_VALUE]) > 1 - ] - ) - - def get_settings(self) -> dict: - return self.settings.__dict__ - - -class MatrixKey(dict): - def __init__(self, settings): - self.settings = settings - - def __str__(self): - return "|".join(f"{k}={self.settings[k]}" for k in sorted(self.settings) if k != "stats") - - def __repr__(self): - return str(self) - - def __hash__(self): - return hash(str(self)) - - -LTS_META_KEYS = [ - "kpi_settings_version", - "lts_schema_version", - "help", - "unit", - "@timestamp", - "value", - "run_id", - "urls", - "test_path", - "ci_engine", - "test_uuid", - "exit_code", - "format", - "full_format", - "ignored_for_regression", - "divisor", - "divisor_unit", - "lower_better", -] - - -class MatrixDefinition: - def __init__(self, is_lts=False): - self.settings = defaultdict(set) - self.import_map = {} - self.processed_map = {} - self.is_lts = is_lts - - def settings_to_key(self, settings): - return MatrixKey(settings) - - def similar_records( - self, - _ref_settings, - ignore_keys, - gathered=False, - rewrite_settings=lambda x: x, - ignore_lts_meta_keys=True, - ): - ref_settings = rewrite_settings(dict(_ref_settings.__dict__)) - - i = 0 - for entry in self.all_records(gathered=gathered): - entry_settings = rewrite_settings(dict(entry.settings.__dict__)) - skip = False - i += 1 - for k, v in ref_settings.items(): - if ignore_lts_meta_keys and k in LTS_META_KEYS: - continue - - if k in ignore_keys: - continue - if entry_settings.get(k, ...) == v: - continue - - skip = True - break - - if not skip: - yield entry - - def filter_records(self, settings, gathered=False): - for entry in self.all_records(gathered=gathered): - skip = False - for k, v in settings.items(): - if entry.settings.__dict__.get(k, ...) == v: - continue - - skip = True - break - - if not skip: - yield entry - - def all_records( - self, settings=None, setting_lists=None, gathered=False - ) -> Iterator[MatrixEntry]: - if not setting_lists: - for e in self.processed_map.values(): - if (gathered and e.is_gathered) or (not gathered and not e.is_gathered): - yield e - return - - for settings_values in sorted( - itertools.product(*setting_lists), key=lambda x: x[0][0] if x else None - ): - settings.update(dict(settings_values)) - - key = self.settings_to_key(settings) - try: - e = self.processed_map[key] - except KeyError: # missing experiment, ignore - continue - - if (gathered and e.is_gathered) or (not gathered and not e.is_gathered): - yield e - - def get_record(self, settings): - key = self.settings_to_key(settings) - - return self.processed_map.get(key, None) - - def count_records(self, settings=None, setting_lists=None): - if settings is None and setting_lists is None: - return sum(1 for e in self.processed_map.values() if not e.is_gathered) - - return sum( - [1 for _ in self.all_records(settings, setting_lists)] - ) # don't use len(list(...)) with a generator, this form is more memory efficient - - def has_records(self, settings, setting_lists): - try: - _first_entry = next( - self.all_records(settings, setting_lists) - ) # raises an exception is the generator is empty - return True - except StopIteration: - return False - - def print_settings_to_log(self): - if not self.processed_map: - return False - - logging.info("Settings matrix:") - - for key, values in self.settings.items(): - if key == "stats": - continue - - self.settings[key] = sorted(values, key=lambda x: (x is MISSING_SETTING_VALUE, x)) - - value_str = ", ".join(map(str, self.settings[key])) - if self.is_lts: - if key == "@timestamp": - value_str = f"<{len(self.settings[key])}x values>" - - if len(value_str) > 1170: - value_str = f"<{len(self.settings[key])} values>" - - logging.info(f"{key:20s}: {value_str}") - - logging.info("---") - - return True - - def uniformize_settings_keys(self): - orig_processed_map = dict(self.processed_map) - - self.processed_map = {} - for entry_key, entry in orig_processed_map.items(): - for settings_key in self.settings.keys(): - if settings_key in entry_key.settings: - continue - - entry_key.settings[settings_key] = MISSING_SETTING_VALUE - self.settings[settings_key].add(MISSING_SETTING_VALUE) - entry.settings.__dict__[settings_key] = MISSING_SETTING_VALUE - - self.processed_map[entry_key] = entry - - -Matrix = MatrixDefinition() -LTS_Matrix = MatrixDefinition(is_lts=True) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/download.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/download.py deleted file mode 100644 index b555b8caa..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/download.py +++ /dev/null @@ -1,176 +0,0 @@ -import logging -import pathlib - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.downloading as downloading -import matrix_benchmarking.store as store -import urllib3 -import yaml - - -def main( - url_file: str = "", - url: str = "", - workload: str = "", - workload_base_dir: str = "", - results_dirname: str = "", - filters: list[str] = [], - do_download: bool = False, - mode: downloading.DownloadModes = None, -): - """ - Download MatrixBenchmarking results. - - Download MatrixBenchmarking results. - - Env: - MATBENCH_URL_FILE - MATBENCH_URL - MATBENCH_WORKLOAD - MATBENCH_WORKLOAD_BASE_DIR - MATBENCH_RESULTS_DIRNAME - MATBENCH_DO_DOWNLOAD - MATBENCH_MODE - - See the `FLAGS` section for the descriptions. - - Args: - url_file: File where the URLs to download are stored. - url: URL that will be downloaded - workload_dir: Name of the workload to execute. (Mandatory.) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - results_dirname: Name of the directory where the results will be stored. Can be set in the benchmark file. (Mandatory.) - do_download: if 'False', list the files that would be downloaded. If 'True', download them. - mode: 'prefer_cache' to download only the cache file, if it exists, or turn to 'mandatory' if it doesn't. - 'important' to download only the important files. - 'all' to download all the files. - """ - - kwargs = dict(locals()) # capture the function arguments - cli_args.setup_env_and_kwargs(kwargs) - cli_args.check_mandatory_kwargs( - kwargs, - ( - "workload", - "results_dirname", - ), - ) - - try: - if not kwargs["mode"]: - kwargs["mode"] = "prefer_cache" - - kwargs["mode"] = downloading.DownloadModes(kwargs["mode"]) - except ValueError: - logging.error(f"Invalid download mode: {kwargs['mode']}") - return 1 - - if not do_download: - logging.warning("Running in DRY MODE (pass the flag --do-download to disable it)") - - def download_an_entry(an_entry, workload_store): - destdir = an_entry["dest_dir"] - source_url = urllib3.util.url.parse_url(an_entry["url"]) - - settings = an_entry["settings"] - - base_dir = pathlib.Path(source_url.path) - dest_dir = pathlib.Path(kwargs["results_dirname"]) / destdir - - logging.info("Downloading an entry ...") - logging.info(f"- src url: {source_url}") - logging.info(f"- dest dir: {dest_dir}") - - scrapper_class = downloading.get_scrapper_class(source_url) - - if do_download: - dest_dir.mkdir(parents=True, exist_ok=True) - with open(dest_dir / "source_url", "w") as f: - print(source_url, file=f) - - with open(dest_dir / "settings.from_url_file.yaml", "w") as f: - yaml.dump(settings, f, indent=4) - - def download(dl_mode): - scrapper = scrapper_class( - workload_store, source_url, base_dir, dest_dir, do_download, dl_mode - ) - scrapper.scrape() - - def download_prefer_cache(): - if hasattr(workload_store, "load_cache"): - download(downloading.DownloadModes.CACHE_ONLY) - successes = 0 - failed = 0 - for cache_file in dest_dir.glob("**/" + workload_store.CACHE_FILENAME): - try: - logging.info(f"Reloading {cache_file} ...") - if workload_store.load_cache(cache_file.parent): - logging.info( - f"Validation the cache of '{cache_file.parent}' succeeded." - ) - successes += 1 - except FileNotFoundError as e: - logging.warning( - f"Validation of the cache of '{cache_file.parent}' failed: {e} :/" - ) - failed += 1 - - logging.info(f"Downloaded {successes} valid cached directories") - if not failed: - # all good, no need to knowload the IMPORTANT files - return - - logging.warning(f"Downloaded {failed} INVALID cached directories") - - logging.info("PREFER_CACHE downloading failed. Switching to IMPORTANT mode.") - - # download or reload from cache worked failed, try again with the important files - download(downloading.DownloadModes.IMPORTANT) - - if do_download and kwargs["mode"] == downloading.DownloadModes.PREFER_CACHE: - download_prefer_cache() - else: - download(kwargs["mode"]) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="download") - - workload_store = store.load_workload_store(kwargs) - - if kwargs["url_file"]: - try: - with open(kwargs["url_file"]) as f: - data = yaml.safe_load(f) - except FileNotFoundError as e: - logging.error(f"Could not open the URL file: {e}") - return 1 - - elif kwargs["url"]: - data = dict(download=[dict(url=kwargs["url"], dest_dir="expe/from_url", settings={})]) - - else: - logging.error("Please specify an URL file or an URL") - return 1 - - try: - for entry in data["download"]: - if "files" not in entry: - # download entry is here, download it - download_an_entry(entry, workload_store) - continue - - # download entries are in another file, process it - for filename in entry["files"]: - with open(pathlib.Path(kwargs["url_file"]).parent / filename) as f: - download_file_data = yaml.safe_load(f) - for download_file_entry in download_file_data: - download_an_entry(download_file_entry, workload_store) - - except KeyboardInterrupt: - print("Interrupted :/") - return 1 - - return 0 - - return cli_args.TaskRunner(run) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/download_lts.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/download_lts.py deleted file mode 100644 index b90326eaf..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/download_lts.py +++ /dev/null @@ -1,160 +0,0 @@ -import datetime -import json -import logging -import pathlib -import sys - -import matrix_benchmarking.cli_args as cli_args -import opensearchpy -import yaml -from opensearchpy import OpenSearch - -LTS_ANCHOR_NAME = "source.lts.yaml" - - -def main( - opensearch_host: str = "", - opensearch_port: str = "", - opensearch_username: str = "", - opensearch_password: str = "", - opensearch_index: str = "", - lts_results_dirname: str = "", - filters: list[str] = [], - max_records: int = 10000, - force: bool = None, - clean: bool = None, -): - """ - Download MatrixBenchmark result from OpenSearch - - Download MatrixBenchmark from Long-Term Storage, expects OpenSearch credentials and configuration to be available either in the enviornment or in an env file. - - Args: - opensearch_host: hostname of the OpenSearch instance - opensearch_port: port of the OpenSearch instance - opensearch_username: username of the OpenSearch instance - opensearch_password: password of the OpenSearch instance - opensearch_index: the OpenSearch index where the LTS payloads are stored (Mandatory) - - lts_results_dirname: The directory to place the downloaded LTS results files. - filters: If provided, only download the experiments matching the filters. Eg: {"image_name": "1.2"}. (Optional.) - max_records: Maximum number of records to retrieve from the OpenSearch instance. 10,000 is the largest number possible without paging (Optional.) - force: Ignore the presence of the anchor file before downloading the results (Optional.) - clean: Delete all the existing '.json' files in the lts-results-dirname before downloading the results (Optional.) - """ - - kwargs = dict(locals()) # capture the function arguments - - optionals_flags = ["filters", "max_records", "force", "clean"] - safe_flags = [ - "filters", - "lts_results_dirname", - "opensearch_index", - "max_records", - "force", - "clean", - ] - - cli_args.update_env_with_env_files() - cli_args.update_kwargs_with_env(kwargs) - cli_args.check_mandatory_kwargs( - kwargs, - mandatory_flags=[k for k in kwargs.keys() if k not in optionals_flags], - sensitive_flags=[k for k in kwargs.keys() if k not in safe_flags], - ) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="download-lts") - - client = connect_opensearch_client(kwargs) - - return download( - client, - kwargs.get("opensearch_index"), - kwargs.get("filters"), - pathlib.Path(kwargs.get("lts_results_dirname")), - kwargs.get("max_records"), - kwargs.get("force"), - kwargs.get("clean"), - ) - - return cli_args.TaskRunner(run) - - -def connect_opensearch_client(kwargs): - auth = (kwargs["opensearch_username"], kwargs["opensearch_password"]) - - client = OpenSearch( - hosts=[{"host": kwargs["opensearch_host"], "port": kwargs["opensearch_port"]}], - timeout=60, - http_compress=True, - http_auth=auth, - use_ssl=True, - verify_certs=False, - ssl_assert_hostname=False, - ssl_show_warn=False, - ) - - return client - - -def download(client, opensearch_index, filters, lts_results_dirname, max_records, force, clean): - lts_dir_anchor = lts_results_dirname / LTS_ANCHOR_NAME - if lts_dir_anchor.exists(): - if not force: - logging.critical(f"{lts_dir_anchor} already exists, cannot continue.") - return 1 - logging.warning(f"{lts_dir_anchor} already exists, ignoring it as --force flag is set.") - - if clean: - cnt = -1 - for cnt, existing_json_file in enumerate(lts_results_dirname.glob("*.json")): - if existing_json_file.name.startswith("."): - continue - - existing_json_file.unlink() - if cnt == -1: - logging.info("No json to cleanup in the LTS results directory.") - else: - logging.info(f"Removed {cnt + 1} json file in the LTS results directory.") - - lts_results_dirname.mkdir(exist_ok=True, parents=True) - - logging.info(f"Querying OpenSearch {opensearch_index} ...") - - query = {"size": max_records} - - # Restrict the results to specific settings - if filters: - query["query"] = { - "bool": {"must": [{"term": {f"{k}.keyword": v}} for k, v in filters.items()]} - } - - try: - search = client.search(body=query, index=opensearch_index) - except opensearchpy.exceptions.NotFoundError: - logging.fatal(f"Fatal: Index '{opensearch_index}' does not exist, cannot proceed.") - sys.exit(123) # No medium found - - logging.info(f"Saving OpenSearch {opensearch_index} results ...") - - with open(lts_dir_anchor, "w") as f: - anchor = dict( - index=opensearch_index, - date=datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ"), - filters=filters, - ) - yaml.dump(anchor, f, indent=4) - print("", file=f) # add EOL - - saved = 0 - lts_results_dirname.mkdir(exist_ok=True, parents=True) - for hit in search["hits"]["hits"]: - with open(lts_results_dirname / f"{opensearch_index}_{hit['_id']}.json", "w") as f: - entry = hit["_source"] - json.dump(entry, f, indent=4) - print("", file=f) # add EOL - - saved += 1 - - logging.info(f"Saved {saved} OpenSearch {opensearch_index} results.") diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/__init__.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/__init__.py deleted file mode 100644 index 06540bc1a..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/__init__.py +++ /dev/null @@ -1,131 +0,0 @@ -import enum -import logging -import pathlib - -import requests - - -class DownloadModes(enum.Enum): - CACHE_ONLY = "cache_only" - PREFER_CACHE = "prefer_cache" - IMPORTANT = "important" - ALL = "all" - - -def get_scrapper_class(url): - # lazy loading to avoid circular imports - - if "openshiftapps.com" in url.host: - logging.info("OpenShift CI scrapping detected.") - - import matrix_benchmarking.downloading.scrape.ocp_ci as ocp_ci_scrapper - - return ocp_ci_scrapper.ScrapOCPCiArtifacts - - if "ci.app-svc-perf.corp.redhat.com" in url.host: - logging.info("Middleware CI scrapping detected.") - - import matrix_benchmarking.downloading.scrape.middleware_ci as middleware_ci_scrapper - - return middleware_ci_scrapper.ScrapMiddlewareCiArtifacts - - if url.scheme == "s3": - logging.info("S3 scrapping detected.") - - import matrix_benchmarking.downloading.scrape.s3 as s3_scrapper - - return s3_scrapper.ScrapS3 - - raise ValueError(f"Download url '{url}' not supported :/") - - -class BaseScapper: - def __init__( - self, workload_store, source_url, base_dir, result_local_dir, do_download, download_mode - ): - self.workload_store = workload_store - self.source_url = source_url - self.base_dir = base_dir - self.result_local_dir = result_local_dir - self.do_download = do_download - self.download_mode = download_mode - self.download_only_cache = self.download_mode in ( - DownloadModes.PREFER_CACHE, - DownloadModes.CACHE_ONLY, - ) - - def download_file(self, filepath_rel, local_filename, depth, handler): - raise NotImplementedError() - - def scrape(self, current_href=None, depth=0, test_found=False): - raise NotImplementedError() - - def is_test_directory(self, filenames): - test_dir_filename = getattr(self.workload_store, "TEST_DIR_FILE", None) # optional - - if "skip" in filenames: - return False - - for filename in filenames: - if test_dir_filename: - # don't merge these two ifs, that changes the semantic ... - if filename == test_dir_filename: - return True - elif filename in ("exit_code", "settings", "settings.yaml"): - return True - - return False - - def has_cache_file(self, filenames, test_found, depth): - if test_found and depth != 0: - return False - - return self.workload_store.CACHE_FILENAME in filenames - - def handle_file(self, filepath_rel, local_filename, depth, handler=None): - if local_filename.exists(): - # file already downloaded, skip it - logging.info(f"{' ' * depth}File: {filepath_rel}: EXISTS") - return - - result_filepath_rel = pathlib.Path(*filepath_rel.parts[-(depth + 1) :]) - - mandatory = self.workload_store.is_mandatory_file(result_filepath_rel) - - cache = self.workload_store.is_cache_file(result_filepath_rel) - - if self.download_mode == DownloadModes.CACHE_ONLY and not cache and not mandatory: - logging.info(f"{' ' * depth}File: {filepath_rel}: NOT CACHE/MANDATORY") - return # file isn't important, do not download it - - important = ( - True - if cache or mandatory - else self.workload_store.is_important_file(result_filepath_rel) - ) - - only_important_files = self.download_mode in ( - DownloadModes.IMPORTANT, - DownloadModes.PREFER_CACHE, - ) - if only_important_files and not important: - logging.info(f"{' ' * depth}File: {filepath_rel}: NOT IMPORTANT") - return # file isn't important, do not download it - - self.download_file(filepath_rel, local_filename, depth, handler) - - -class BaseHttpScapper(BaseScapper): - def download_file(self, filepath_rel, local_filename, depth, handler): - url = f"{self.source_url.scheme}://{self.source_url.host}/{self.base_dir}/{filepath_rel}" - - if not self.do_download: - return - - local_filename.parent.mkdir(parents=True, exist_ok=True) - - with requests.get(url, stream=True, verify=False) as r: - r.raise_for_status() - with open(local_filename, "wb") as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/middleware_ci.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/middleware_ci.py deleted file mode 100644 index 6cd1e93f6..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/middleware_ci.py +++ /dev/null @@ -1,62 +0,0 @@ -import logging -import pathlib - -import requests -import urllib3 - -urllib3.disable_warnings() - -from bs4 import BeautifulSoup - -from .. import BaseHttpScapper - - -class ScrapMiddlewareCiArtifacts(BaseHttpScapper): - def scrape(self, current_href=None, depth=0, test_found=False): - url = f"{self.source_url.scheme}://{self.source_url.host}/{current_href if current_href else self.base_dir}" - - r = requests.get(url, verify=False) - s = BeautifulSoup(r.text, "html.parser") - - links = [ - svg.parent.next_sibling.find("a") for svg in s.find_all("svg", {"class": "icon-sm"}) - ] - filenames = [link.text for link in links] - cache_found = False - - if not test_found and self.is_test_directory(filenames): - depth = 0 - test_found = True - logging.info(f"Found a test directory at {url}") - - if self.has_cache_file(filenames, test_found, depth): - cache_found = True - - for link in links: - new_href = (current_href or self.base_dir) / pathlib.Path(link.attrs["href"]) - - svg_class = link.parent.previous_sibling.find("svg").attrs[ - "class" - ] # always exists, because of the way links/link is located - - if "icon-folder" in svg_class: - # link to a directory, recurse into it - - if cache_found and self.download_only_cache: - logging.info( - f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}: SKIP (cache found)" - ) - continue - - logging.info(f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}") - self.scrape(new_href, depth=depth + 1, test_found=test_found) - - elif "icon-document" in svg_class: - # link to a file, defer to the child class to decide what to do with it - rel_path = new_href.relative_to(self.base_dir) - local_filename = self.result_local_dir / rel_path - - self.handle_file(rel_path, local_filename, depth) - else: - # ignore - pass diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/ocp_ci.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/ocp_ci.py deleted file mode 100644 index 643dc82ca..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/ocp_ci.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging -import pathlib - -import requests -import urllib3 - -urllib3.disable_warnings() - -from bs4 import BeautifulSoup - -from .. import BaseHttpScapper - - -class ScrapOCPCiArtifacts(BaseHttpScapper): - def scrape(self, current_href=None, depth=0, test_found=False): - url = f"{self.source_url.scheme}://{self.source_url.host}/{current_href if current_href else self.base_dir}" - - r = requests.get(url) - s = BeautifulSoup(r.text, "html.parser") - - filenames = [(pathlib.Path(link.attrs["href"]).name) for link in s.find_all("a")] - cache_found = False - - if not test_found and self.is_test_directory(filenames): - depth = 0 - test_found = True - logging.info(f"Found a test directory at {url}") - - if self.has_cache_file(filenames, test_found, depth): - cache_found = True - - for link in s.find_all("a"): - new_href = pathlib.Path(link.attrs["href"]) - - img = link.find() - if img is None and link.text == "gsutil": - # link to download gsutil, ignore - continue - - img = link.find("img") - if not img: - # link without an image, so not a file/directory. Ignore. - # (must be a link to `gsutil` or `gcloud storage`) - continue - - img_src = img["src"] - if img_src == "/icons/back.png": - # link going to the parent directory, ignore - continue - - if img_src == "/icons/dir.png": - # link to a directory, recurse into it - - if cache_found and self.download_only_cache: - logging.info( - f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}: SKIP (cache found)" - ) - continue - - logging.info(f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}") - self.scrape(new_href, depth=depth + 1, test_found=test_found) - - elif img_src == "/icons/file.png": - # link to a file, defer to the child class to decide what to do with it - rel_path = new_href.relative_to(self.base_dir) - local_filename = self.result_local_dir / rel_path - - self.handle_file(rel_path, local_filename, depth) - else: - # ignore - pass diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/s3.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/s3.py deleted file mode 100644 index 2684a28c2..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/downloading/scrape/s3.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging -import pathlib - -import boto3 - -from .. import BaseScapper - - -class ScrapS3(BaseScapper): - def download_file(self, filepath_rel, local_filename, depth, handler): - logging.info(f"{' ' * depth}File: {filepath_rel}: DOWNLOAD") - - if not self.do_download: - return - - s3_client = handler - - local_filename.parent.mkdir(parents=True, exist_ok=True) - - s3_client.download_file( - self.source_url.host, str(self.base_dir / filepath_rel).strip("/"), local_filename - ) - - if not local_filename.exists(): - raise RuntimeError(f"Something unexpected happened, {local_filename} does not exist :/") - - def scrape(self, current_href=None, depth=0, test_found=False, handler=None): - if handler is None: - session = boto3.Session() # use the env/default settings to login into AWS - handler = boto3.client("s3") - - s3_client = handler - - current_dir = current_href or self.base_dir - response = s3_client.list_objects_v2( - Bucket=self.source_url.host, Prefix=str(current_dir).strip("/") + "/", Delimiter="/" - ) - - filenames = [pathlib.Path(entry["Key"]).name for entry in response.get("Contents", [])] - dirnames = [ - pathlib.Path(entry["Prefix"]).name for entry in response.get("CommonPrefixes", []) - ] - - cache_found = False - - if not test_found and self.is_test_directory(filenames): - depth = 0 - test_found = True - logging.info(f"Found a test directory at {current_dir}") - - if self.has_cache_file(filenames, test_found, depth): - cache_found = True - - for filename in filenames: - new_href = (current_href or self.base_dir) / filename - rel_path = new_href.relative_to(self.base_dir) - local_filename = self.result_local_dir / rel_path - - self.handle_file(rel_path, local_filename, depth, handler) - - for dirname in dirnames: - new_href = (current_href or self.base_dir) / dirname - - if cache_found and self.download_only_cache: - logging.info( - f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}: SKIP (cache found)" - ) - continue - - logging.info(f"{' ' * depth}Directory: {new_href.relative_to(self.base_dir)}") - self.scrape(new_href, depth=depth + 1, test_found=test_found) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/common.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/common.py deleted file mode 100644 index 75a26d1f1..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/common.py +++ /dev/null @@ -1,176 +0,0 @@ -import datetime -import logging -import os -import pathlib -import sys -import types - -import jinja2 -import matrix_benchmarking.exec.kube as kube -import matrix_benchmarking.exec.prom as prom -import yaml - - -def apply_yaml_template(template_file, settings): - with open(template_file) as f: - template_str = f.read() - - template = jinja2.Template(template_str).render(settings) - - return template, list(yaml.safe_load_all(template)) - - -def is_connected(): - try: - version_dict = kube.custom.get_cluster_custom_object( - "config.openshift.io", "v1", "clusterversions", "version" - ) - logging.debug("Connected to the cluster.") - return True - - except Exception as e: - logging.error(e) - logging.warning("Is the Kubernetes cluster reachable? Aborting.") - - return False - - -def prepare_settings(): - settings = types.SimpleNamespace() - - logging.info("Settings:") - for arg in sys.argv[1:]: - k, _, v = arg.partition("=") - settings.__dict__[k] = v - logging.info(f"- {k} = {v}") - - return settings - - -_artifacts_dir = None - - -def create_artifact_dir(benchmark_name): - global _artifacts_dir - - if sys.stdout.isatty(): - base_dir = pathlib.Path("/tmp") / ( - "matrix-benchmarking" + datetime.datetime.today().strftime("%Y%m%d") - ) - base_dir.mkdir(exist_ok=True) - current_length = len(list(base_dir.glob("*__*"))) - _artifacts_dir = base_dir / f"{current_length:03d}__benchmarking__run_{benchmark_name}" - _artifacts_dir.mkdir(exist_ok=True) - else: - _artifacts_dir = pathlib.Path(os.getcwd()) - - src_dir = _artifacts_dir / "src" - src_dir.mkdir(exist_ok=True) - - logging.info(f"Saving artifacts files into {_artifacts_dir}") - - -def save_artifact(content, filename, *, is_src=False, mode="w"): - dest_dir = _artifacts_dir - if is_src: - dest_dir /= "src" - - with open(dest_dir / filename, mode) as out_f: - out_f.write(content) - - -def save_system_artifacts(): - logging.info("-----") - logging.info("Collecting system artifacts ...") - - def save_nodes(): - nodes = kube.corev1.list_node() - nodes_dict = nodes.to_dict() - - try: - del nodes_dict["metadata"]["managed_fields"] - except KeyError: - pass # ignore - - try: - del nodes_dict["status"]["images"] - except KeyError: - pass # ignore - - save_artifact(yaml.dump(nodes_dict), "nodes.yaml") - - def save_cluster_version(): - logging.info("Saving OpenShift version ...") - - version_dict = kube.custom.get_cluster_custom_object( - "config.openshift.io", "v1", "clusterversions", "version" - ) - try: - del version_dict["metadata"]["managedFields"] - except KeyError: - pass # ignore - - save_artifact(yaml.dump(version_dict), "ocp_version.yaml") - - save_nodes() - save_cluster_version() - - -def prepare_prometheus(): - prom_data = types.SimpleNamespace() - - with time_it("restart_prometheus"): - prom_data.handler = prom.restart_prometheus() - prom_data.start_ts = prom.query_current_ts(prom_data.handler) - - return prom_data - - -def finalize_prometheus(prom_data=None): - if not prom_data: - prom_data = types.SimpleNamespace() - prom_data.handler = prom.get_handler() - - prom_data.end_ts = prom.query_current_ts(prom_data.handler) - - def save_db_raw(): - logging.info("Capturing Prometheus database ...") - tgz_content = prom.dump_prometheus_db_raw(prom_data.handler) - - dest_tgz = "prometheus_db.tgz" - - save_artifact(tgz_content, dest_tgz, mode="wb") - db_size = len(tgz_content) - logging.info(f"Prometheus database saved into '{dest_tgz}' ({db_size / 1024 / 1024:.0f}MB)") - - def save_db_json(): - logging.info("Capturing Prometheus metrics ...") - db_json = prom.dump_prometheus_db_json( - prom_data.handler, prom_data.start_ts, prom_data.end_ts - ) - with time_it("prometheus json to file"): - save_artifact(yaml.dump(db_json), "prometheus_db.json") - - logging.info("Capturing Prometheus metrics ... done.") - - with time_it("save_prometheus_db_raw"): - save_db_raw() - # with time_it("save_prometheus_db_json"): save_db_json() - - -class time_it: - def __init__(self, msg): - self.start_time = None - self.msg = msg - - def __enter__(self): - self.start_time = datetime.datetime.now() - - def __exit__(self, ex_type, ex_value, ex_tb): - end_time = datetime.datetime.now() - seconds = (end_time - self.start_time).seconds - if seconds < 180: - time_str = f"{seconds} seconds" - else: - time_str = f"{seconds / 60:.1f} minutes" - logging.info(f"time_it: {self.msg}: %s", time_str) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/kube.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/kube.py deleted file mode 100644 index 1e7dcf682..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/kube.py +++ /dev/null @@ -1,12 +0,0 @@ -import kubernetes.client -import kubernetes.config -import kubernetes.utils - -kubernetes.config.load_kube_config() - -corev1 = kubernetes.client.CoreV1Api() -appsv1 = kubernetes.client.AppsV1Api() -batchv1 = kubernetes.client.BatchV1Api() -custom = kubernetes.client.CustomObjectsApi() - -k8s_client = kubernetes.client.ApiClient() diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/prom.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/prom.py deleted file mode 100644 index 86d2b40d7..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/exec/prom.py +++ /dev/null @@ -1,233 +0,0 @@ -import base64 -import datetime -import logging -import math -import time -import types - -logging.info("Importing prometheus_api_client ...") -import matrix_benchmarking.exec.kube as kube -import prometheus_api_client - - -def _get_secret_token(): - logging.info("Prometheus: Fetching the monitoring secret token ...") - - secrets = kube.corev1.list_namespaced_secret(namespace="openshift-user-workload-monitoring") - for secret in secrets.items: - name = secret.metadata.name - if not name.startswith("prometheus-user-workload-token"): - continue - return base64.b64decode(secret.data["token"]).decode("ascii") - - return "" - - -def _get_thanos_hostname(): - logging.info("Prometheus: Fetching the route URL ...") - - thanos_querier_route = kube.custom.get_namespaced_custom_object( - group="route.openshift.io", - version="v1", - namespace="openshift-monitoring", - plural="routes", - name="thanos-querier", - ) - - return thanos_querier_route["spec"]["host"] - - -def _has_user_monitoring(): - logging.info("Prometheus: Checking if user-monitoring is enabled ...") - try: - monitoring_cm = kube.corev1.read_namespaced_config_map( - namespace="openshift-monitoring", name="cluster-monitoring-config" - ) - cfg = monitoring_cm.data["config.yaml"] - - return "enableUserWorkload: true" in cfg - except kube.kubernetes.client.exceptions.ApiException as e: - if e.reason != "Not Found": - raise e - return False - except KeyError: - return False - - -def _get_prometheus_podinfo(): - PROM_POD_LABEL = "app.kubernetes.io/component=prometheus" - - podinfo = types.SimpleNamespace() - podinfo.container = "prometheus" - - podinfo.namespace = "openshift-monitoring" - - pods = kube.corev1.list_namespaced_pod( - namespace=podinfo.namespace, label_selector=PROM_POD_LABEL - ) - - if not pods.items: - raise RuntimeError(f"Pod {label} not found in {namespace} ...") - - podinfo.podname = pods.items[0].metadata.name - - return podinfo - - -def _get_PrometheusConnect(handler): - import urllib3 - - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - headers = dict(Authorization=f"Bearer {handler.token}") - return prometheus_api_client.PrometheusConnect( - url=f"https://{handler.host}", - headers=headers, - disable_ssl=True, - ) - - -def _exec_in_pod(namespace, podname, container, cmd): - exec_command = ["/bin/sh", "-c", cmd] - - return kube.k8s_stream( - kube.corev1.connect_get_namespaced_pod_exec, - namespace=namespace, - command=exec_command, - name=podname, - container=container, - stderr=False, - stdin=False, - stdout=True, - tty=False, - ) - - -def do_query(handler, api_cmd, **data): - if not handler.token: - raise RuntimeError("Prometheus token not available ...") - - url = f"https://{handler.host}/api/v1/{api_cmd}" - encoded_data = urllib.parse.urlencode(data) - url += "?" + encoded_data - - curl_cmd = f"curl --silent -k '{url}' --header 'Authorization: Bearer {handler.token}'" - - resp = _exec_in_pod( - handler.prom_podinfo.namespace, - handler.prom_podinfo.podname, - handler.prom_podinfo.container, - curl_cmd, - ) - - result = json.loads(resp.replace("'", '"')) - - if result["status"] == "success": - return result["data"] - - -def get_handler(): - if not _has_user_monitoring(): - logging.error( - """User monitoring not enabled. See https://docs.openshift.com/container-platform/4.10/monitoring/enabling-monitoring-for-user-defined-projects.html#enabling-monitoring-for-user-defined-projects_enabling-monitoring-for-user-defined-projects""" - ) - raise SystemExit(1) - - handler = types.SimpleNamespace() - - handler.token = _get_secret_token() - handler.host = _get_thanos_hostname() - handler.prom_podinfo = _get_prometheus_podinfo() - handler.prom_connect = _get_PrometheusConnect(handler) - - return handler - - -def restart_prometheus(): - prom_podinfo = _get_prometheus_podinfo() - handler = get_handler() - - logging.info("Stopping Prometheus Pod ...") - kube.corev1.delete_namespaced_pod(prom_podinfo.podname, prom_podinfo.namespace) - - while True: - logging.info("Waiting for Prometheus Pod to be recreated ...") - - try: - handler.prom_podinfo = _get_prometheus_podinfo() - except RuntimeError: - time.sleep(1) - continue - - break # Prometheus Pod exists - - logging.info("Prometheus Pod is running again.") - - logging.info("Waiting for Prometheus to respond properly ...") - while True: - try: - if query_current_ts(handler) is not None: - break - except kube.kubernetes.client.exceptions.ApiException as e: - if "500" not in e.reason: - raise e - except json.JSONDecodeError: - pass # ignore - - time.sleep(1) - - return handler - - -def dump_prometheus_db_raw(handler): - resp = _exec_in_pod( - handler.prom_podinfo.namespace, - handler.prom_podinfo.podname, - handler.prom_podinfo.container, - "tar cvzf - /prometheus | base64", - ) - - return base64.standard_b64decode(resp) - - -def query_current_ts(handler): - try: - metric = handler.prom_connect.get_current_metric_value( - metric_name="cluster:memory_usage:ratio" - ) - return metric[0]["value"][0] - except IndexError: - return None - - -def query_values(handler, metric, ts_start, ts_stop): - minutes = math.ceil( - ( - datetime.datetime.fromtimestamp(ts_stop) - datetime.datetime.fromtimestamp(ts_start) - ).seconds - / 60 - ) - return handler.prom_connect.custom_query( - query=f"{metric}[{minutes}m]", params=dict(time=ts_stop) - ) - - -def dump_prometheus_db_json(handler, start_ts, stop_ts): - all_metrics = handler.prom_connect.all_metrics() - - def chunker(seq, size): - return (seq[pos : pos + size] for pos in range(0, len(seq), size)) - - results = [] - current_group = all_metrics[1] - last_metric = all_metrics[-1] - for metric in all_metrics: - current_group += f"|{metric}" - # when the request size is longer that 21300 chars, we get an error 'HTTP Status Code 400 | Bad request' - if len(current_group) < 20000 and metric != last_metric: - continue - - results.append(query_values(handler, f'{{__name__=~"{current_group}"}}', start_ts, stop_ts)) - logging.info(f"{len(results)}) Found {len(results[-1])} new values") - current_group = metric - - return results diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/generate_lts_schema.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/generate_lts_schema.py deleted file mode 100644 index ed3d16a72..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/generate_lts_schema.py +++ /dev/null @@ -1,147 +0,0 @@ -import contextlib -import json -import logging -import pathlib -import sys - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.store as store - - -@contextlib.contextmanager -def smart_open(filename=None): - if filename and filename != "-": - fh = open(filename, "w") - else: - fh = sys.stdout - - try: - yield fh - finally: - if fh is not sys.stdout: - fh.close() - - -def main(workload: str = "", workload_base_dir: str = "", file: str = "-"): - """ - Generate the JSON LTS schema of the workload - - Generate the JSON LTS schema of the workload - - Args: - workload: Name of the workload to execute. (Mandatory) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - file: Name of the file to export the schema to, "-" to output to stdout. (Default is "-") - """ - - kwargs = dict(locals()) - - cli_args.setup_env_and_kwargs(kwargs) - cli_args.check_mandatory_kwargs(kwargs, ["workload"]) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="export_schema") - workload_store = store.load_workload_store(kwargs) - - schema = store.get_lts_schema() - if not schema: - logging.error(f"No LTS schema registered for workload '{workload}', cannot export it.") - sys.exit(1) - - schema_dict = schema.schema() - - filename = kwargs.get("file") - - with smart_open(filename) as f: - json.dump(schema_dict, f, indent=4) - print("", file=f) # add EOL at EOF - - if filename != "-": - filepath = pathlib.Path(filename) - with smart_open(filepath.parent / "opensearch_mapping.json") as f: - json.dump(create_opensearch_mapping(schema_dict), f, indent=4) - print("", file=f) # add EOL at EOF - - return cli_args.TaskRunner(run) - - -TYPE_MAP = dict( - string="text", - number="float", - integer="long", - array="text", -) - -FORMAT_TYPE_MAP = { - "date-time": "date", -} - - -def create_opensearch_mapping(json_schema): - definitions = {} - - os_mapping = {} - - def process(path, entry, dest): - if defs := entry.get("definitions"): - for def_name, def_value in defs.items(): - definitions[f"{path}/definitions/{def_name}"] = def_value - - if def_name == "PrometheusValue": - def_value["properties"]["values"]["index"] = False - def_value["properties"]["values"]["type"] = "text" - - if ref := entry.get("$ref"): - entry.pop("$ref") - entry |= definitions[ref].copy() - - if ref := entry.get("items", {}).get("$ref"): - entry.pop("items") - entry |= definitions[ref].copy() - - if path == "#" and "type" in entry: - entry.pop("type") - - if path.endswith("values"): - dest["index"] = False - entry["type"] = "text" - - if path != "#" and "title" in entry and "type" not in entry: - entry["type"] = "object" - entry["index"] = False - - for k, v in entry.items(): - if k == "$ref": - logging.fatal("Found a stray $ref :/") - sys.exit(1) - elif k == "type": - if fmt := entry.get("format"): - v = FORMAT_TYPE_MAP.get(fmt, v) - - dest[k] = TYPE_MAP.get(v, v) # convert or passthrough - elif k in ( - "additionalProperties", - "title", - "format", - "default", - "required", - "pattern", - "value", # ignore, not supported - "description", - "enum", - "items", - "definitions", # ignore, already processed - ): - continue # ignore - elif isinstance(v, dict): - processed_dict = {} - if k == "regression_results": - return - process(f"{path}/{k}", v, processed_dict) - dest[k] = processed_dict - else: - dest[k] = v - - process("#", json_schema, os_mapping) - - return os_mapping diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/main.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/main.py deleted file mode 100644 index 3add6bda4..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/main.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 - -import logging -import sys - -logging.basicConfig(format="%(levelname)s | %(message)s", level=logging.INFO) - -try: - import fire -except ModuleNotFoundError: - logging.error( - "MatrixBenchmarking requires the Python `fire` package, see requirements.txt for a full list of requirements" - ) - sys.exit(1) - -import matrix_benchmarking.analyze_lts -import matrix_benchmarking.benchmark -import matrix_benchmarking.download -import matrix_benchmarking.download_lts -import matrix_benchmarking.generate_lts_schema -import matrix_benchmarking.parse -import matrix_benchmarking.upload_lts -import matrix_benchmarking.visualize - - -class MatrixBenchmarking: - """ - Commands for launching MatrixBenchmarking - """ - - def __init__(self): - self.benchmark = matrix_benchmarking.benchmark.main - self.visualize = matrix_benchmarking.visualize.main - self.parse = matrix_benchmarking.parse.main - self.download = matrix_benchmarking.download.main - self.upload_lts = matrix_benchmarking.upload_lts.main - self.download_lts = matrix_benchmarking.download_lts.main - self.generate_lts_schema = matrix_benchmarking.generate_lts_schema.main - self.analyze_lts = matrix_benchmarking.analyze_lts.main - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - # Launch CLI, get a runnable - runnable = None - runnable = fire.Fire(MatrixBenchmarking()) - - # Run the actual workload - if hasattr(runnable, "run"): - return runnable.run() - else: - # CLI didn't resolve completely - either by lack of arguments - # or use of `--help`. This is okay. - pass - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/matrix.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/matrix.py deleted file mode 100644 index f4bdff221..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/matrix.py +++ /dev/null @@ -1,318 +0,0 @@ -import itertools -import logging -import os -import pathlib -import subprocess -import sys -import types - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import yaml - - -class Matrix: - def __init__(self, yaml_desc): - self.yaml_desc = yaml_desc - - def run( - self, - ): - tracker = types.SimpleNamespace() - tracker.expe_cnt = types.SimpleNamespace() - tracker.expe_cnt.total = 0 - tracker.expe_cnt.current_idx = 0 - tracker.expe_cnt.executed = 0 - tracker.expe_cnt.recorded = 0 - tracker.expe_cnt.errors = 0 - tracker.dry = not cli_args.kwargs["run"] - - expe_ran = [] - - expe_to_run = cli_args.kwargs["expe_to_run"] - if isinstance(expe_to_run, str): - expe_to_run = expe_to_run.split(",") - - for expe in expe_to_run: - if not expe or expe.startswith("_"): - logging.info(f"Skip disabled expe '{expe}'") - continue - - stop = self.do_run_expe(tracker, expe) - if stop: - break - expe_ran.append(expe) - - logging.info( - f"Ran {len(expe_ran)} {'matrices' if len(expe_ran) > 1 else 'matrix'}: {', '.join(expe_ran)}" - ) - logging.info(f"Out of {tracker.expe_cnt.total} experiments configured:") - if tracker.dry: - logging.info(f"- {tracker.expe_cnt.executed} would have been executed,") - else: - logging.info( - f"- {tracker.expe_cnt.executed} {'has' if tracker.expe_cnt.executed == 1 else 'have'} been executed," - ) - logging.info( - f"- {tracker.expe_cnt.recorded} {'was' if tracker.expe_cnt.recorded == 1 else 'were'} already recorded," - ) - logging.info(f"- {tracker.expe_cnt.errors} failed.") - - return tracker.expe_cnt.errors - - def do_run_expe(self, tracker, expe): - logging.info("setup()") - - try: - yaml_expe = self.yaml_desc["expe"][expe] - except KeyError: - logging.error(f"Cannot run experiment '{expe}': experiment matrix not defined.") - return True - - if not isinstance(yaml_expe, dict): - raise RuntimeError(f"Expe '{expe}' content should be a mapping ...") - - context = types.SimpleNamespace() - context.settings = types.SimpleNamespace() - - context.expe = expe - context.expe_dir = pathlib.Path(".") / cli_args.kwargs["results_dirname"] / context.expe - - context.path_tpl = cli_args.kwargs["path_tpl"] - context.script_tpl = cli_args.kwargs["script_tpl"] - context.remote_mode = cli_args.kwargs["remote_mode"] - context.stop_on_error = cli_args.kwargs["stop_on_error"] - - context.common_settings = self.yaml_desc.get("common_settings", {}) - - if not tracker.dry and context.remote_mode and tracker.expe_cnt.current_idx == 0: - print( - r"""#! /bin/bash - -set -x - -if ! [[ -d "$1" ]]; then - echo "FATAL: \$1 should point to the result directory" - exit 1 -fi -RESULTS_DIR="$(realpath "$1")" - -if ! [[ -d "$2" ]]; then - echo "FATAL: \$2 should point to 'exec' directory " - exit 1 -fi -EXEC_DIR="$(realpath "$2")" -""", - file=sys.stderr, - ) - - settings = dict(context.common_settings or {}) - settings.update(yaml_expe) - - all_settings_items = [ - [(name, value) for value in (values if isinstance(values, list) else [values])] - for name, values in settings.items() - ] - - tracker.expe_cnt.total += sum(1 for _ in itertools.product(*all_settings_items)) - - if not tracker.dry and not context.remote_mode: - os.makedirs(context.expe_dir, exist_ok=True) - - stop = self.do_run_matrix(tracker, all_settings_items, context, yaml_expe) - if stop: - return stop - - logging.info("#") - logging.info(f"# Finished with '{context.expe}'") - logging.info("#") - logging.info("") - - return False - - def do_run_matrix(self, tracker, all_settings_items, context, yaml_expe): - for settings_items in itertools.product(*all_settings_items): - settings = dict(settings_items) - - tracker.expe_cnt.current_idx += 1 - - path_tpl = context.path_tpl - expe_path_tpl = settings.get("--path-tpl") - if expe_path_tpl: - del settings["--path-tpl"] - path_tpl = expe_path_tpl - - if path_tpl is None: - raise ValueError( - ".--path-tpl or .expe[].--path-tpl must be provided." - ) - - if "extra" in settings: - extra = settings["extra"] - del settings["extra"] - if isinstance(extra, dict): - raise ValueError(f"'extra' is a dict, does it contain a ':'? ({extra})") - for kv in extra.split(", "): - if "=" not in kv: - raise ValueError(f"Invalid 'extra' setting: '{extra}' ('{kv}' has no '=')") - k, v = kv.split("=") - settings[k.strip()] = v.strip() - - key = common.Matrix.settings_to_key(settings | dict(expe=context.expe)) - - if key in common.Matrix.processed_map or key in common.Matrix.import_map: - logging.info( - f"experiment {tracker.expe_cnt.current_idx}/{tracker.expe_cnt.total} already recorded, skipping." - ) - location = ( - common.Matrix.processed_map[key].location - if key in common.Matrix.processed_map - else common.Matrix.import_map[key].location - ) - - logging.info(f"> {location.relative_to(context.expe_dir.parent)}") - logging.info("") - tracker.expe_cnt.recorded += 1 - continue - - try: - bench_common_path = path_tpl.format(**settings, settings=settings).replace("/", "_") - except KeyError as e: - logging.error( - f"cannot apply the path template '{path_tpl}': key '{e.args[0]}' missing from {settings}" - ) - tracker.expe_cnt.errors += 1 - if context.stop_on_error: - return True - continue - - parent_dir = (context.expe_dir / bench_common_path).parent # may or may not exist - parent_dir_matches = sorted(parent_dir.glob("*__*")) - next_id = ( - (int(list(parent_dir_matches[-1].name.split("__"))[0]) + 1) - if parent_dir_matches - else 0 - ) - - bench_common_pathname = f"{next_id:03d}__{bench_common_path}" - - context.bench_dir = pathlib.Path(context.expe) / bench_common_pathname - context.bench_fullpath = context.expe_dir / bench_common_pathname - - if not tracker.dry and not context.remote_mode: - os.makedirs(context.bench_fullpath) - - logging.info("---" * 5) - logging.info("") - logging.info("") - - logging.info(f"running {tracker.expe_cnt.current_idx}/{tracker.expe_cnt.total}") - for k, v in settings.items(): - logging.info(f" {k}: {v}") - try: - ret = self.execute_benchmark(settings, context, tracker) - except KeyboardInterrupt: - logging.error("Stopping on keyboard interrupt.") - return True - - if ret != None and not ret: - tracker.expe_cnt.errors += 1 - if context.stop_on_error: - logging.warning("Stopping on error.") - return True - - tracker.expe_cnt.executed += 1 - return False - - def execute_benchmark(self, settings, context, tracker): - if not tracker.dry and not context.remote_mode: - with open(context.bench_fullpath / "settings.yaml", "w") as out_f: - yaml.dump(settings, out_f) - - for test_file, _content in self.yaml_desc.get("test_files", {}).items(): - content = ( - _content - if isinstance(_content, str) - else yaml.dump(_content, default_flow_style=False, sort_keys=False) - ) - - with open(context.bench_fullpath / test_file, "w") as f: - print(content, file=f) - - try: - script = context.script_tpl.format(**settings) - except KeyError as e: - logging.error( - f"cannot apply the script template '{context.script_tpl}': key '{e.args[0]}' missing from {settings}" - ) - tracker.expe_cnt.errors += 1 - if context.stop_on_error: - return False - - return None - - cmd_fullpath = f"{pathlib.Path(os.getcwd()) / script} &> >(tee run.log)" - - if tracker.dry: - try: - results_dir = context.bench_fullpath.relative_to(context.expe_dir.parent.parent) - except ValueError: # (fullpath )is not in the subpath of (expe_dir) - results_dir = context.expe_dir - - logging.info(f"""\n -Results: {results_dir} -Command: {script} ---- -""") - - return None - elif context.remote_mode: - settings_str = yaml.dump(settings) - print( - f""" -echo "Expe {tracker.expe_cnt.current_idx}/{tracker.expe_cnt.total}" -CURRENT_DIRNAME="${{RESULTS_DIR}}/{context.bench_dir}" - -if [[ "$(cat "$CURRENT_DIRNAME/exit_code" 2>/dev/null)" != 0 ]]; then - mkdir -p "$CURRENT_DIRNAME" - cd "$CURRENT_DIRNAME" - [[ "$$CURRENT_DIRNAME" ]] && rm -rf -- "$CURRENT_DIRNAME"/* - echo -e "{settings_str}" > ./settings.yaml - echo "$(date) Running expe {tracker.expe_cnt.current_idx}/{tracker.expe_cnt.total}" - ${{EXEC_DIR}}/{script} - echo "$?" > ./exit_code -else - echo "Already recorded in $CURRENT_DIRNAME." -fi -""", - file=sys.stderr, - ) - return None - - # no need to check here if ./exit_code exists and == 0, - # we wouldn't reach this step if it did - # (whereas the remote_mode generated script can be executed multiple times) - - logging.info(f"cd {context.bench_fullpath}") - logging.info(cmd_fullpath) - try: - proc = subprocess.run( - cmd_fullpath, - cwd=context.bench_fullpath, - shell=True, - stdin=subprocess.PIPE, - executable="/bin/bash", - ) - except KeyboardInterrupt as e: - logging.info("") - logging.info("KeyboardInterrupt registered.") - raise e - - ret = proc.returncode - # /!\ ^^^^^^^^^^^^^^^ blocks until the process terminates - - logging.info(f"exit code: {ret}") - with open(context.bench_fullpath / "exit_code", "w") as out_f: - print(f"{ret}", file=out_f) - - return ret == 0 diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/models.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/models.py deleted file mode 100644 index c371413c1..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/models.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import annotations - -import datetime -import datetime as dt -import enum -import inspect -from typing import Annotated, Optional - -import pydantic -from pydantic import UUID4, BaseModel, ConfigDict, Field - -SEMVER_REGEX = r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" - - -class ExclusiveModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - __force_annotation: int # workaround for https://github.com/python/cpython/issues/95532, for AllOptional 'annotations.update(base.__annotations__)' to work - - -class AllOptional(pydantic._internal._model_construction.ModelMetaclass): - def __new__(self, name, bases, namespaces, **kwargs): - annotations = namespaces.get("__annotations__", {}) - for base in bases: - if hasattr(base, "__annotations__"): - annotations.update(base.__annotations__) - for field in annotations: - if not field.startswith("__"): - annotations[field] = Optional[annotations[field]] - namespaces["__annotations__"] = annotations - return super().__new__(self, name, bases, namespaces, **kwargs) - - -class PSAPEnum(str, enum.Enum): - def _generate_next_value_(name, start, count, last_values): - return name.replace("_", " ") - - def __str__(self) -> str: - return self.value - - -class EntryStatus(PSAPEnum): - Valid = enum.auto() - Invalid = enum.auto() - - -class Empty(ExclusiveModel): ... - - -class Metadata(ExclusiveModel): - status: EntryStatus = Field(default=EntryStatus.Valid) - - start: dt.datetime - end: dt.datetime - settings: dict[str, str | int] - test_uuid: UUID4 - urls: list[str] | None - exit_code: int | None - - -class PrometheusValue(ExclusiveModel): - metric: dict[str, str] - values: dict[int, float] - - -PrometheusValues = list[PrometheusValue] - -PrometheusNamedMetricValues = dict[str, PrometheusValues] - - -class PrometheusMetric(ExclusiveModel): - query: str - data: PrometheusValues - - -def create_schema_field(schema_name): - return Annotated[str, Field(pattern=f"^urn:{schema_name}:{SEMVER_REGEX}$")] - - -class KPI(ExclusiveModel): - unit: str - help: str - timestamp: datetime.datetime = Field(..., alias="@timestamp") - value: float | int | list[float] | list[int] - test_uuid: UUID4 - lower_better: bool | None - - status: EntryStatus = Field(default=EntryStatus.Valid) - - ci_engine: str | None - run_id: str | None - test_path: str | None - - urls: dict[str, str] | None - - # - ignored_for_regression: bool | None = Field(default=None, exclude=True) - format: str | None = Field(default=None, exclude=True) - full_format: str | None = Field(default=None, exclude=True) - divisor: float | None = Field(default=None, exclude=True) - divisor_unit: str | None = Field(default=None, exclude=True) - # - - def __str__(self): - labels = { - k: v - for k, v in self.__dict__.items() - if k not in ("unit", "help", "timestamp", "value") - } - labels_str = ", ".join(f'{k}="{v}"' for k, v in labels.items()) - - return f"""# HELP {self.help}, in {self.unit} -__NAME__{{{labels_str}}} {self.value} {self.unit}""" - - -def IgnoredForRegression(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - - if name not in mod.KPIs: - raise KeyError(f"@IgnoredForRegression should come before @KPIMetadata() for {name}") - - mod.KPIs[name]["ignored_for_regression"] = True - - return fct - - -def LowerBetter(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - - if name not in mod.KPIs: - raise KeyError(f"@LowerBetter should come before @KPIMetadata() for {name}") - - if mod.KPIs[name].get("lower_better") is not None: - raise KeyError(f"@LowerBetter should not be used with @HigherBetter for {name}") - - mod.KPIs[name]["lower_better"] = True - - return fct - - -def HigherBetter(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - - if name not in mod.KPIs: - raise KeyError(f"@HigherBetter should come before @KPIMetadata() for {name}") - - if mod.KPIs[name].get("lower_better") is not None: - raise KeyError(f"@HigherBetter should not be used with @LowerBetter for {name}") - - mod.KPIs[name]["lower_better"] = False - - return fct - - -# -# Receives a divisor to apply to the value before formatting it -# 100, "MB" -# -def FormatDivisor(divisor: float, unit: str, format: str = None): - def decorator(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - - if name not in mod.KPIs: - raise KeyError(f"@FormatDivisor should come before @KPIMetadata() for {name}") - - if format: - if mod.KPIs[name].get("format") is not None: - raise KeyError(f"@FormatDivisor(fmt) should not be used with @Format for {name}") - mod.KPIs[name]["format"] = format - else: - if mod.KPIs[name].get("format") is None: - raise KeyError(f"@FormatDivisor should be used with @Format for {name}") - - mod.KPIs[name]["divisor"] = divisor - mod.KPIs[name]["divisor_unit"] = unit - - return fct - - return decorator - - -# -# Receives a format string as parameter -# eg: "{.2f}" -# -def Format(format: str): - def decorator(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - - if name not in mod.KPIs: - raise KeyError(f"@Format should come before @KPIMetadata() for {name}") - - if mod.KPIs[name].get("full_format"): - raise KeyError(f"@Format should not be used with @FullFormat for {name}") - - mod.KPIs[name]["format"] = format - - return fct - - return decorator - - -# -# Receives a KPI as parameter -# eg: -def FullFormat(format_fct): - def decorator(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - if callable(format_fct) and format_fct.__name__ == "": - raise KeyError(f"@FullFormat does not work with Lambda for {name}") - - if name not in mod.KPIs: - raise KeyError(f"@FullFormat should come before @KPIMetadata() for {name}") - - if mod.KPIs[name].get("format"): - raise KeyError(f"@FullFormat should not be used with @Format for {name}") - - mod.KPIs[name]["full_format"] = format_fct - - return fct - - return decorator - - -def KPIMetadata(help, unit): - def decorator(fct): - mod = inspect.getmodule(fct) - - name = fct.__name__ - if name in mod.KPIs: - raise KeyError(f"Key {name} already exists in module {fct.__module__}.") - - mod.KPIs[name] = dict(help=help, unit=unit) - mod.KPIs[name]["__func__"] = fct - mod.KPIs[name]["ignored_for_regression"] = False - mod.KPIs[name]["format"] = None - mod.KPIs[name]["full_format"] = None - mod.KPIs[name]["divisor"] = None - mod.KPIs[name]["divisor_unit"] = None - return fct - - return decorator - - -def getKPIsModel(name, module_name, KPIs, KPImodel): - model = pydantic.create_model( - name, - __base__=ExclusiveModel, - **dict(zip(KPIs.keys(), [(KPImodel, ...) for _ in range(len(KPIs))], strict=False)), - __module__=module_name, - ) - - def tostr(self): - return "\n".join( - [ - str(getattr(self, name)).replace("__NAME__", name) - for name in self.model_fields.keys() - ] - ) - - model.tostr = tostr - - return model diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/parse.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/parse.py deleted file mode 100644 index 7c2f0a544..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/parse.py +++ /dev/null @@ -1,167 +0,0 @@ -import functools -import json -import logging -import sys - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.store as store - - -def json_dumper(obj, strict=False): - import datetime - import pathlib - - if isinstance(obj, dict): - return obj - - elif hasattr(obj, "toJSON"): - return obj.toJSON() - - elif hasattr(obj, "json"): - return obj.dict(by_alias=True) - - elif hasattr(obj, "__dict__"): - return obj.__dict__ - - if isinstance(obj, datetime.datetime): - return obj.isoformat() - - elif isinstance(obj, pathlib.Path): - return str(obj) - elif not strict: - return str(obj) - else: - raise RuntimeError(f"No default serializer for object of type {obj.__class__}: {obj}") - - -def main( - workload: str = "", - workload_base_dir: str = "", - results_dirname: str = "", - filters: list[str] = [], - clean: bool = False, - run: bool = False, - output_lts: str = "", - output_matrix: str = "", - pretty: bool = True, - lts: bool = False, -): - """ - Run MatrixBenchmarking results parsing. - - Run MatrixBenchmarking results parsing (for troubleshooting). - - Env: - MATBENCH_WORKLOAD - MATBENCH_WORKLOAD_BASE_DIR - MATBENCH_RESULTS_DIRNAME - MATBENCH_FILTERS - MATBENCH_CLEAN - MATBENCH_RUN - - See the `FLAGS` section for the descriptions. - - Args: - workload: Name of the workload to execute. (Mandatory.) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - results_dirname: Name of the directory where the results will be stored. Can be set in the benchmark file. (Mandatory.) - filters: If provided, parse only the experiment matching the filters. Eg: expe=expe1:expe2,something=true. - clean: If 'True', run in cleanup mode. - run: In cleanup mode: if 'False', list the results that would be cleanup. If 'True', execute the cleanup. - output_lts: Output the parsed LTS results into a specified file, or to stdout if '-' is supplied - output_matrix: Output the internal entry matrix into a specified file, or to stdout if '-' is supplied - lts: If 'True', invoke the LTS parser only. - """ - - kwargs = dict(locals()) # capture the function arguments - - cli_args.setup_env_and_kwargs(kwargs) - cli_args.check_mandatory_kwargs( - kwargs, - ( - "workload", - "results_dirname", - ), - ) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="parse_clean") - - if kwargs["clean"]: - logging.info("Running the result directory cleaner...") - - workload_store = store.load_workload_store(kwargs) - - logging.info("Loading results ... ") - - if kwargs.get("lts"): - workload_store.parse_lts_data() - else: - workload_store.parse_data() - - logging.info(f"Loading results: done, found {len(common.Matrix.processed_map)} results") - common.Matrix.uniformize_settings_keys() - - if kwargs["clean"]: - if not kwargs["run"]: - logging.info("Cleaner ran in dry mode. Pass --run to perform the deletion.") - - common.Matrix.print_settings_to_log() - - if kwargs["output_matrix"]: - indent = None - if kwargs["pretty"]: - indent = 4 - - parsed_results = [] - for entry in common.Matrix.processed_map.values(): - parsed_results.append(entry) - - file = None - try: - file = ( - sys.stdout - if kwargs["output_matrix"] == "-" - else open(kwargs["output_matrix"], "w") - ) - json.dump( - parsed_results, - file, - indent=indent, - default=functools.partial(json_dumper, strict=False), - ) - print("", file=file) # add the EOL delimiter after the JSON dump - finally: - if file: - file.close() - - if kwargs["output_lts"]: - indent = None - if kwargs["pretty"]: - indent = 4 - - parsed_results = [] - for payload, _, __ in workload_store.build_lts_payloads(): - parsed_results.append(payload) - - file = None - try: - file = ( - sys.stdout if kwargs["output_lts"] == "-" else open(kwargs["output_lts"], "w") - ) - json.dump( - parsed_results, - file, - indent=indent, - default=functools.partial(json_dumper, strict=False), - ) - print("", file=file) - finally: - if file: - file.close() - - has_result = len(common.Matrix.processed_map) != 0 - return 0 if has_result else 1 - - return cli_args.TaskRunner(run) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/parsing/prom.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/parsing/prom.py deleted file mode 100644 index 1b891ce62..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/parsing/prom.py +++ /dev/null @@ -1,100 +0,0 @@ -import statistics as stats - - -def filter_single(metrics): - if len(metrics) != 1: - raise ValueError(f"filter_single expected to find only one metric. Found {len(metrics)}") - - yield metrics[0] - - -def filter_all(metrics): - yield from metrics - - -def filter_value_in_label(metrics, value, label): - for metric in metrics: - label_value = metric.metric.get(label) - if label_value is None: - continue - if value not in label_value: - continue - - # found it - yield metric - - -def filter_doesnt_have_label(metrics, label): - for metric in metrics: - if label in metric.metric: - continue - - yield metric - - -# --- - - -def mean(metrics, filter_fct): - values = [] - for metric in filter_fct(metrics): - values.append(stats.mean([float(v) for ts, v in metric.values.items()])) - - return values - - -def last(metrics, filter_fct): - values = [] - for metric in filter_fct(metrics): - values.append(float(list(metric.values.values())[-1])) - - return values - - -def max_(metrics, filter_fct): - values = [] - for metric in filter_fct(metrics): - values.append(max([float(v) for ts, v in metric.values.items()])) - - return values - - -# --- - - -def single_max(metrics): - return max_(metrics, filter_single) - - -def single_mean(metrics): - return mean(metrics, filter_single) - - -def single_last(metrics): - return last(metrics, filter_single) - - -# --- - - -def all_max(metrics): - return max_(metrics, filter_all) - - -def max_max(metrics): - return max(max_(metrics, filter_all)) - - -# --- - - -def min_mean(metrics): - return min(mean(metrics, filter_all)) - - -def max_mean(metrics): - return max(mean(metrics, filter_all)) - - -def mean_mean(metrics): - return stats.mean(mean(metrics, filter_all)) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/__init__.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/__init__.py deleted file mode 100644 index 3dc46a35f..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -import re - - -def natural_keys(text): - def atoi(text): - return int(text) if text.isdigit() else text - - return [atoi(c) for c in re.split(r"(\d+)", str(text))] - - -def join(joiner, iterable): - i = iter(iterable) - try: - yield next(i) # First value, or StopIteration - while True: - next_value = next(i) - yield joiner - yield next_value - except StopIteration: - pass - - -def COLORS(idx): - colors = [ - "#1f77b4", # muted blue - "#ff7f0e", # safety orange - "#2ca02c", # cooked asparagus green - "#d62728", # brick red - "#9467bd", # muted purple - "#8c564b", # chestnut brown - "#e377c2", # raspberry yogurt pink - "#7f7f7f", # middle gray - "#bcbd22", # curry yellow-green - "#17becf", # blue-teal - ] - return colors[idx % len(colors)] diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/table_stats.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/table_stats.py deleted file mode 100644 index 6d0de44ec..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/table_stats.py +++ /dev/null @@ -1,571 +0,0 @@ -import logging -import statistics -from collections import defaultdict - -import matrix_benchmarking.common as common -import plotly.graph_objs as go -from dash import dcc -from matrix_benchmarking import plotting - - -def register_all(): - for stat in TableStats.all_stats: - common.Matrix.settings["stats"].add(stat.name) - - if not isinstance(stat, TableStats): - continue - - for entry in common.Matrix.processed_map.values(): - if not entry.is_gathered: - entry.stats[stat.name] = stat.process(entry) - continue - - entry.stats[stat.name] = gathered_stats = [] - for gathered_entry in entry.results: - if not gathered_entry.results: - continue - - gathered_stats.append(stat.process(gathered_entry)) - - -class TableStats: - all_stats = [] - stats_by_name = {} - stats_by_id = {} - graph_figure = None - - @classmethod - def _register_stat(clazz, stat_obj): - logging.info(f"Registering '{stat_obj.name}' ...") - - clazz.all_stats.append(stat_obj) - - if stat_obj.name in clazz.stats_by_name: - raise Exception(f"Duplicated name: {stat_obj.name}") - - clazz.stats_by_name[stat_obj.name] = stat_obj - clazz.stats_by_id[stat_obj.id_name] = stat_obj - - def __init__(self, id_name, name, field, fmt, unit, higher_better, divisor=1, **kwargs): - self.id_name = id_name - self.name = name - self.field = field - self.fmt = fmt - self.unit = unit - self.higher_better = higher_better - self.divisor = divisor - self.kwargs = kwargs - - self.do_process = None - - TableStats._register_stat(self) - - def __str__(self): - return self.name - - def __repr__(self): - return self.id_name - - @classmethod - def Custom(clazz, *args, **kwargs): - obj = clazz(*args, **kwargs) - - def process(entry): - raise RuntimeError("Should not be called ...") - - obj.do_process = process - return obj - - @classmethod - def Value(clazz, *args, **kwargs): - obj = clazz(*args, **kwargs) - obj.do_process = obj.process_value_dev - return obj - - @classmethod - def ValueDev(clazz, *args, **kwargs): - obj = clazz(*args, **kwargs) - obj.do_process = obj.process_value_dev - return obj - - @classmethod - def MeanStd(clazz, *args, **kwargs): - obj = clazz(*args, **kwargs) - obj.do_process = obj.process_mean_std - return obj - - def process(self, entry): - class FutureValue: - def __init__(self): - self.computed = False - self._value = None - self._stdev = None - - @property - def value(myself): - if myself._value is not None: - return myself._value - try: - v = self.do_process(entry) - except Exception as e: - logging.error( - f"Failed to process field '{self.field}' with" - f"{self.do_process.__self__.__class__.__name__}.{self.do_process.__name__}:" - ) - logging.error(f"{e.__class__.__name__}:{e}") - raise e - - try: - myself._value, *myself._stdev = v - except TypeError: # cannot unpack non-iterable ... object - myself._value = v - - return myself._value - - @property - def stdev(myself): - if myself._value is not None: - _not_used = myself.value # force trigger the computation - - return myself._stdev - - def __str__(myself): - if myself.value is None: - return "N/A" - - val = f"{myself.value:{self.fmt}}{self.unit}" - if not myself.stdev: - pass - elif len(myself.stdev) == 1: - if myself.stdev[0] is not None: - val += f" +/- {myself.stdev[0]:{self.fmt}}{self.unit}" - elif len(myself.stdev) == 2: - if myself.stdev[0] is not None: - val += f" + {myself.stdev[0]:{self.fmt}}" - if myself.stdev[1] is not None: - val += f" - {myself.stdev[1]:{self.fmt}}" - val += str(self.unit) - return val - - return FutureValue() - - def process_gathered_value_dev(self, entry): - values = [self.process_value_dev(gathered_entry)[0] for gathered_entry in entry.results] - values = [v for v in values if v is not None] - - mean = statistics.mean(values) / self.divisor - - stdev = statistics.stdev(values) if len(values) > 2 else 0 - - return mean, (stdev / self.divisor) - - def process_value_dev(self, entry): - if entry.is_gathered: - return self.process_gathered_value_dev(entry) - - value = self.field(entry) - - if value is None: - return None, None - - if isinstance(value, list): - logging.warning( - f"{entry.location}.results.{self.field} is a list of length {len(value)}" - ) - value = value[0] - - dev_field = self.kwargs.get("dev_field") - - dev_value = dev_field(entry) if dev_field else 0 - - return value, dev_value - - def process_mean_std(self, entry): - values = self.field(entry) - - if not isinstance(value, list): - logging.warning( - f"{entry.location}.results.{self.field} is NOT a list of length ({values})" - ) - values = [values] - - if not values: - logging.warning("{entry.location}.results.{self.field} is an empty list") - return 0, 0 - - mean = statistics.mean(values) / self.divisor - - stdev = statistics.stdev(values) if len(values) > 2 else 0 - - return mean, (stdev / self.divisor) - - def do_hover(self, meta_value, variables, figure, data, click_info): - ax = figure["data"][click_info.idx]["xaxis"] - - xaxis = "xaxis" + (ax[1:] if ax != "x" else "") - yaxis = figure["layout"]["yaxis"]["title"]["text"] - - try: - xaxis_name = figure["layout"][xaxis]["title"]["text"] - except KeyError: - xaxis_name = "" - - props = ", ".join([click_info.x, click_info.legend, xaxis_name]).split(", ") - value = f"{yaxis}: {click_info.y:.2f}" - - entry, msg = TableStats.props_to_hoverlink(variables, props, value) - - graph = self.entry_to_hovergraph(entry) if entry else "" - - return [*msg, graph] - - def entry_to_hovergraph(self, entry): - # - # needs to be changed - # - if not hasattr(entry, "tables"): - return "" - - for table_def, (table_name, table_rows) in entry.tables.items(): - if table_name != self.table and not ( - self.table.startswith("?.") and table_name.endswith(self.table[1:]) - ): - continue - - table_simple_name = table_name.split(".")[1] - - def get_values(field_name): - row_id = table_def.partition("|")[2].split(";").index(field_name) - return [row[row_id] for row in table_rows if row[row_id] is not None] - - try: - x = get_values(f"{table_simple_name}.msg_ts") - except ValueError: - try: - x = get_values("time") - except ValueError: - logging.warning( - f"props_to_hovergraph: Cannot find {table_simple_name}.msg_ts or time field ..." - ) - return "" - - field = self.field if not isinstance(self.field, tuple) else self.field[1] - - y = get_values(field) - from . import graph - - try: - x = graph.GraphFormat.as_timestamp(x, y) - except ValueError: # year 50265933 is out of range - x = graph.GraphFormat.as_timestamp([_x / 1000000 for _x in x], y) - - fig = go.Figure(data=go.Scatter(x=x, y=y)) - - fig.update_layout(yaxis_title=f"{self.name} ({self.unit})") - return dcc.Graph(figure=fig) - - return "Table not found ..." - - @staticmethod - def props_to_hoverlink(variables, props, value): - for prop in props: - try: - k, v = prop.split("=") - except ValueError: - continue # not enough values to unpack (expected 2, got 1) - variables[k] = v - - entry = common.Matrix.get_record(variables) - if not entry: - return None, "Error: record not found in matrix ..." - - loc = entry.location - msg = [f"{loc} ⇒ {value}"] - - return entry, msg - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - cfg_plot_mode = cfg.get("stats.var_length", "") - - if cfg_plot_mode: - var_length = int(cfg_plot_mode) - logging.info(f"Using var_length={var_length} instead of {len(variables)}") - else: - var_length = len(variables) - - data = [] - layout = go.Layout() - layout.hovermode = "closest" - layout.meta = (dict(name=self.name),) - - if len(variables) == 0: - layout.title = "Select at least 1 variable parameter..." - return [{"data": data, "layout": layout}] - - *second_vars, legend_var = ordered_vars - second_vars.reverse() - - layout.title = f"{self.name} vs " + " x ".join(ordered_vars[:-1]) + " | " + ordered_vars[-1] - layout.yaxis = dict(title=self.name + f" ({self.unit})") - layout.plot_bgcolor = "rgb(245,245,240)" - subplots = {} - - if second_vars: - subplots_var = second_vars[-1] - - showticks = len(second_vars) == 2 - for i, subplots_key in enumerate( - sorted(variables[subplots_var], key=plotting.natural_keys) - ): - subplots[subplots_key] = f"x{i + 1}" - ax = f"xaxis{i + 1}" - layout[ax] = dict( - title=f"{subplots_var}
{subplots_key}", - type="category", - showticklabels=showticks, - ) - - else: - subplots_var = None - subplots[subplots_var] = "x1" - layout["xaxis1"] = dict( - type="category", showticklabels=True, tickfont=dict(size=18), tickangle=45 - ) - - x = defaultdict(list) - y = defaultdict(list) - y_err = defaultdict(list) - legend_keys = set() - legend_names = set() - legends_visible = [] - subplots_used = set() - - for entry in common.Matrix.all_records(settings, setting_lists): - if self.name not in entry.stats: - logging.info(f"Stat '{self.name}' not found for entry '{entry.location}'") - continue - - x_key = ", ".join( - [f"{v}={settings[v]}" for v in reversed(second_vars) if v != subplots_var] - ) - legend_name = f"{legend_var}={settings[legend_var]}" - legend_key = (legend_name, settings[subplots_var] if subplots_var else None) - - if var_length > 3 and x[legend_key]: - prev_first_setting = x[legend_key][-1].partition(" ")[0] - first_setting = x_key.partition(" ")[0] - - if prev_first_setting != first_setting: - x[legend_key].append(None) - y[legend_key].append(None) - y_err[legend_key].append(None) - - legend_keys.add(legend_key) - if not x_key: - x_key = legend_key[0].split("=")[1] - - legend_names.add(legend_name) - x[legend_key].append(x_key) - - if entry.is_gathered: - y[legend_key].append(entry.stats[self.name][0].value) - y_err[legend_key].append(entry.stats[self.name][0].stdev) - else: - y[legend_key].append(entry.stats[self.name].value) - y_err[legend_key].append(entry.stats[self.name].stdev) - - # --- - def prepare_histogram(legend_key, color): - plot_args["type"] = "bar" - plot_args["marker"] = dict(color=color) - - def plot_histogram_err(legend_key): - error_y = plot_args["error_y"] = dict(type="data", visible=True) - error_y["array"] = [err[0] for err in y_err[legend_key]] - - if len(y_err[legend_key][0]) == 2: - error_y["arrayminus"] = [err[1] for err in y_err[legend_key]] - - # --- - - def prepare_scatter(legend_key, color): - if var_length < 5: - plot_args["type"] = "line" - plot_args["line"] = dict(color=color) - else: - plot_args["mode"] = "markers" - plot_args["marker"] = dict(color=color) - - def prepare_scatter_short_err(legend_key): - y_err_above = [] - y_err_below = [] - for _y, _y_error in zip(y[legend_key], y_err[legend_key], strict=False): - # above == below iff len(_y_error) == 1 - if None in (_y, _y_error): - continue - - y_err_above.append(_y + _y_error[0]) - y_err_below.append(_y - _y_error[-1]) - - y_err_data = y_err_above + list(reversed(y_err_below)) - x_err_data = x[legend_key] + list(reversed(x[legend_key])) - - return x_err_data, y_err_data - - def prepare_scatter_long_err(legend_key): - y_err_data = [] - x_err_data = [] - - x_err_current = [] - y_err_above = [] - y_err_below = [] - - for _x, _y, _y_error in zip( - x[legend_key] + [None], - y[legend_key] + [None], - y_err[legend_key] + [None], - strict=False, - ): - if _x is not None: - if None not in (_y, _y_error): - # above == below iff len(_y_error) == 1 - y_err_above.append(_y + _y_error[0]) - y_err_below.append(_y - _y_error[-1]) - else: - y_err_above.append(None) - y_err_below.append(None) - x_err_current.append(_x) - continue - - x_err_data += ( - x_err_current + list(reversed(x_err_current)) + [x_err_current[0], None] - ) - - y_err_data += y_err_above + list(reversed(y_err_below)) + [y_err_above[0], None] - x_err_current = [] - y_err_above = [] - y_err_below = [] - - return x_err_data, y_err_data - - def plot_scatter_err(legend_key, err_data, y_max): - x_err_data, y_err_data = err_data - - data.append( - go.Scatter( - x=x_err_data, - y=y_err_data, - legendgroup=legend_name + ("(stdev)" if var_length >= 4 else ""), - showlegend=False, - hoverinfo="skip", - fill="toself", - fillcolor="rgba(0,100,80,0.2)", - line_color="rgba(0,0,0,0)", - xaxis=ax, - name=legend_name + (" (stdev)" if var_length >= 4 else ""), - ) - ) - - return max([yval for yval in [y_max] + y_err_data if yval is not None]) - - y_max = 0 - legend_keys = sorted(list(legend_keys), key=plotting.natural_keys) - legend_names = sorted(list(legend_names), key=plotting.natural_keys) - - DO_LOCAL_SORT = True - for legend_key in legend_keys: - legend_name, subplots_key = legend_key - ax = subplots[subplots_key] - has_err = any(y_err[legend_key]) - - color = plotting.COLORS(list(legend_names).index(legend_name)) - plot_args = dict() - - if var_length <= 2: - prepare_histogram(legend_key, color) - else: - prepare_scatter(legend_key, color) - - if has_err and var_length < 5: - if var_length <= 2: - err_data = plot_histogram_err(legend_key) - else: - if var_length < 4: - err_data = prepare_scatter_short_err(legend_key) - else: - err_data = prepare_scatter_long_err(legend_key) - - y_max = plot_scatter_err(legend_key, err_data, y_max) - - if var_length >= 5 and DO_LOCAL_SORT: - # sort x according to y's value order - x[legend_key] = [ - _x - for _y, _x in sorted( - zip(y[legend_key], x[legend_key], strict=False), - key=lambda v: (v[0] is None, v[0]), - ) - ] - # sort y by value (that may be None) - y[legend_key].sort(key=lambda x: (x is None, x)) - if not layout.title.text.endswith(" (sorted)"): - layout.title.text += " (sorted)" - - # if 2 >= var_length > 5: - # need to sort and don't move the None location - # need to sort yerr as well - - showlegend = legend_name not in legends_visible - if showlegend: - legends_visible.append(legend_name) - subplots_used.add(ax) - - y_max = max([yval for yval in [y_max] + y[legend_key] if yval is not None]) - - data.append( - dict( - **plot_args, - x=x[legend_key], - y=y[legend_key], - legendgroup=legend_name, - xaxis=ax, - name=legend_name, - showlegend=showlegend, - hoverlabel={"namelength": -1}, - ) - ) - - do_sort = bool(cfg.get("stats.sort_bar", False)) - if do_sort and var_length <= 2: - layout["xaxis"].categoryorder = "trace" - - def get(_name): - for trace in data: - if trace["name"] != _name: - continue - return trace - - for _y, _x in sorted(zip(y.values(), x.keys(), strict=False)): - name = _x[0] - trace = get(name) - data.remove(trace) - data.append(trace) - - if var_length > 2: - # force y_min = 0 | y_max = max visible value (cannot set only y_min) - # if var_length <= 2: - # bar plot start from 0, y_max hard to compute with error bars - layout.yaxis.range = [0, y_max] - - for i, ax in enumerate(sorted(subplots_used, key=lambda x: int(x[1:]))): - axis = "xaxis" + ax[1:] - layout[axis].domain = [i / len(subplots_used), (i + 1) / len(subplots_used)] - - layout.legend.traceorder = "normal" - - # --- - - fig = go.Figure(data) - fig.update_layout(layout) - fig.update_layout(title_x=0.5) - return fig, [""] diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/__init__.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/__init__.py deleted file mode 100644 index 46023e911..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/__init__.py +++ /dev/null @@ -1,600 +0,0 @@ -import datetime -import importlib -import logging -import sys -import traceback -import types -import urllib.parse - -logging.info("Loading dash ...") -import dash -import flask -import plotly.graph_objs as go -from dash import dcc, html -from dash.dependencies import ClientsideFunction, Input, Output, State - -logging.info("Loading dash ... done") - -from matrix_benchmarking import plotting -from matrix_benchmarking.common import Matrix -from matrix_benchmarking.plotting.table_stats import TableStats - -NB_GRAPHS = 3 -GRAPH_IDS = [f"graph-{i}" for i in range(NB_GRAPHS)] -TEXT_IDS = [f"graph-{i}-txt" for i in range(NB_GRAPHS)] - - -def configure(kwargs, workload_store): - workload_package = workload_store.__package__.removesuffix(".store") - module = f"{workload_package}.plotting" - logging.info(f"Loading {module} module ...") - - plotting_module = importlib.import_module(module) - logging.info(f"Loading {module} module ... done.") - - if hasattr(plotting_module, "register"): - plotting_module.register() - - -def get_permalink(args, full=False): - settings = dict(zip(Matrix.settings.keys(), args[: len(Matrix.settings)], strict=False)) - - def val(k, v): - if isinstance(v, list): - return "&".join(f"{k}={vv}" for vv in v) - else: - return f"{k}={v}" - - search = "?" + "&".join( - val(k, v) - for k, v in settings.items() - if v not in ("---", None) and (full or len(Matrix.settings[k]) != 1) - ) - *_, custom_cfg, custom_cfg_saved, settings_order, custom_cfg_saved_state = args - - if settings_order: - search += f"&settings-order={'|'.join(settings_order)}" - - if custom_cfg_saved or custom_cfg: - lst = custom_cfg_saved[:] if custom_cfg_saved else [] - if custom_cfg and custom_cfg not in lst: - lst.insert(0, custom_cfg) - - search += ("&" + "&".join([f"cfg={cfg}" for cfg in lst])) if lst else "" - - return search - - -def sanitize_setting_key(key): - # dash.exceptions.InvalidComponentIdError: The element `list-settings-name.i` contains `.` in its ID. - # Characters `.`, `{` are not allowed in IDs. - return key.replace(".", "-").replace("{", "(").replace("}", ")") - - -def build_layout(search, serializing=False): - defaults = urllib.parse.parse_qs(search.split("?", maxsplit=1)[-1]) if search else {} - - matrix_controls = [html.B("Settings:", id="lbl_settings"), html.Br()] - serial_settings = [] - for key, values in Matrix.settings.items(): - options = [{"label": i, "value": i} for i in sorted(values, key=plotting.natural_keys)] - - attr = {} - if key == "stats": - attr["multi"] = True - - elif len(values) == 1: - attr["disabled"] = True - attr["value"] = options[0]["value"] - else: - options.insert(0, {"label": "[ all ]", "value": "---"}) - attr["searchable"] = False - - if key.startswith("@") and "" in values: - attr["value"] = "" - elif key == "experiment" and "current" in values: - attr["value"] = "current" - else: - attr["value"] = "---" - - try: - default_value = defaults[key] - attr["value"] = default_value[0] if len(default_value) == 1 else default_value - except KeyError: - pass - - if serializing: - attr["disabled"] = True - serial_settings.append(attr["value"]) - serial_stats_position = len(serial_settings) - 1 - - tag = dcc.Dropdown( - id=f"list-settings-{sanitize_setting_key(key)}", - options=options, - **attr, - clearable=False, - ) - - matrix_controls += [html.Span(f"{key}: ", id=f"label_{sanitize_setting_key(key)}"), tag] - - cfg_data = defaults.get("cfg", []) - cfg_children = list([html.P(e) for e in cfg_data]) - - config = [ - html.B("Configuration:", id="config-title"), - html.Br(), - dcc.Input( - id="custom-config", - placeholder="Config settings", - debounce=True, - style={"width": "100%"}, - ), - html.Div(id="custom-config-saved", children=cfg_children, **{"data-label": cfg_data}), - ] - - permalink_download = [ - html.P( - [ - html.A("Permalink", href="", id="permalink"), - " ", - html.A("Download", href="", id="download", target="_blank"), - ] - ) - ] - - try: - settings_order = defaults["settings-order"][0].split("|") - settings_order_children = html.Ol([html.Li(setting) for setting in settings_order]) - settings_order_data = {"data-order": settings_order} - except KeyError: - settings_order_children = "" - settings_order_data = {} - aspect = [html.Div(settings_order_children, id="settings-order", **settings_order_data)] - - control_children = matrix_controls - - if not serializing: - control_children += config + permalink_download + aspect - else: - control_children += [ - html.I(["Saved on ", str(datetime.datetime.today()).rpartition(":")[0]]) - ] - settings_order = defaults.get("settings-order", [""])[0] - - if settings_order: - settings_order = settings_order.split("|") - - permalink = ( - "/matrix/" - + get_permalink( - ( - serial_settings # [Input('list-settings-'+sanitize_setting_key(key), "value") for key in Matrix.settings] - + [""] # custom-config useless here - + [cfg_data] - + [settings_order] - + [] - ), - full=True, - ) - ) - - control_children += [ - html.P(["from ", html.A("this page", target="_blank", href=permalink), "."]) - ] - - graph_children = [] - stats = defaults.get("stats", []) - if serializing: - for idx, stats_name in enumerate(stats): - logging.info(f"Generate {idx}) {stats_name}") - try: - table_stat = TableStats.stats_by_name[stats_name] - except KeyError: - logging.error(f"Stats '{stats_name}' does not exist. Skipping it.") - continue - - graph_children += [ - dcc.Graph(id=table_stat.id_name, style={}, config=dict(showTips=False)), - html.P(id=table_stat.id_name + "-txt"), - ] - - current_serial_settings = [e for e in serial_settings] - current_serial_settings[serial_stats_position] = [stats_name] - - figure_text = TableStats.graph_figure( - *( - current_serial_settings # [Input('list-settings-'+sanitize_setting_key(key), "value") for key in Matrix.settings] - + [0] # Input("lbl_settings", "n_clicks") - + defaults.get("settings-order", [[]]) # Input('settings-order', 'data-order') - + [None] # Input('config-title', 'n_clicks') | None->not clicked yet - + [""] # Input('custom-config', 'value') - + [""] # Input('custom-config-saved', 'data-label') - + [defaults.get("cfg", [""])] # State('custom-config-saved', 'data-label') - ) - ) - - graph, text = graph_children[-2:] - if text: - logging.info("==>" + str(text)) - - graph.figure = figure_text[0] - graph.style["height"] = "100vh" - graph.style["height"] = f"{100 / (min(NB_GRAPHS, len(stats))):.2f}vh" - if not graph.figure: - graph.style["display"] = "none" - - text.children = figure_text[1] - else: - for idx, graph_id in enumerate(GRAPH_IDS): - display = "none" if idx >= len(stats) else "block" - graph_children += [ - dcc.Graph(id=graph_id, style={"display": display}, config=dict(showTips=False)), - html.P(id=graph_id + "-txt"), - ] - - graph_children += [html.Div(id="text-box:clientside-output")] - - return html.Div( - [ - html.Div(children=control_children, className="two columns"), - html.Div(children=graph_children, id="graph-box", className="ten columns"), - html.P(id="graph-hover-info"), - ] - ) - - -def build_callbacks(app): - # Dash doesn't support creating the callbacks AFTER the app is running, - # can the Matrix callback IDs are dynamic (base on the name of the settings) - - if not Matrix.settings: - logging.error("Matrix empty, cannot build its GUI") - return - - @app.server.route("/matrix/dl") - def download_graph(): - search = (b"?" + flask.request.query_string).decode("ascii") - layout = build_layout(search, serializing=True) - - import dill - - data = dill.dumps(layout) - query = urllib.parse.parse_qs(search[1:]) - - fname = ( - "__".join(TableStats.stats_by_name[stat_name].id_name for stat_name in query["stats"]) - if query.get("stats") - else "nothing" - ) - - resp = flask.Response(data, mimetype="application/octet-stream") - resp.headers["Content-Disposition"] = f'attachment; filename="{fname}.dill"' - - return resp - - app.clientside_callback( - ClientsideFunction(namespace="clientside", function_name="resize_graph"), - Output("text-box:clientside-output", "children"), - [Input("permalink", "href"), Input("list-settings-stats", "value")], - ) - - @app.callback( - [ - Output("custom-config-saved", "children"), - Output("custom-config-saved", "data-label"), - Output("custom-config", "value"), - ], - [Input("config-title", "n_clicks")], - [State("custom-config-saved", "data-label"), State("custom-config", "value")], - ) - def save_config(*args): - title_click, data, value = args - if data is None: - data = [] - - if not value: - return dash.no_update, dash.no_update, "" - if value in data: - return dash.no_update, dash.no_update, "" - - if value.startswith("_"): - if value[1:] not in data: - logging.warning( - f"tried to remove '{value[1:]}' but it's not in '{', '.join(data)}'" - ) - return dash.no_update, dash.no_update, dash.no_update - data.remove(value[1:]) - else: - k, _, v = value.partition("=") - - for d in data[:]: - if d.startswith(k + "="): - data.remove(d) - if v: - data.append(value) - - return list([html.P(e) for e in data]), data, "" - - @app.callback( - [Output("settings-order", "children"), Output("settings-order", "data-order")], - [Input(f"label_{sanitize_setting_key(key)}", "n_clicks") for key in Matrix.settings] - + [Input("settings-order", "n_clicks")], - [State("settings-order", "data-order")], - ) - def varname_click(*args): - settings_order = args[-1] - - try: - triggered_id = dash.callback_context.triggered[0]["prop_id"] - except IndexError: - triggered_id = None # nothing triggered the script (on multiapp load) - - if not settings_order: - settings_order = list(Matrix.settings.keys()) - - if triggered_id == "settings-order.n_clicks": - settings_order.append(settings_order.pop(0)) - elif triggered_id and triggered_id != ".": # label_keyframe-period.n_clicks - setting_name = triggered_id.partition("_")[-1].rpartition(".")[0] - if setting_name in settings_order: - settings_order.remove(setting_name) - settings_order.insert(0, setting_name) - - try: - settings_order.remove("stats") - except ValueError: - pass - - return html.Ol([html.Li(setting_name) for setting_name in settings_order]), settings_order - - @app.callback( - Output("graph-hover-info", "children"), - [Input(graph_id, "clickData") for graph_id in GRAPH_IDS], - [State(graph_id, "figure") for graph_id in GRAPH_IDS] - + [State(f"list-settings-{sanitize_setting_key(key)}", "value") for key in Matrix.settings], - ) - def display_hover_data(*args): - hoverData = args[:NB_GRAPHS] - - try: - triggered_id = dash.callback_context.triggered[0]["prop_id"] - except IndexError: - return # nothing triggered the script (on multiapp load) - if triggered_id == ".": - return - - pos = int(triggered_id.rpartition(".")[0].split("-")[1]) - data = hoverData[pos] - - figure = args[NB_GRAPHS : 2 * NB_GRAPHS][pos] - variables = dict(zip(Matrix.settings.keys(), args[2 * NB_GRAPHS :], strict=False)) - - if not figure: - return "Error, figure not found ..." - - click_info = types.SimpleNamespace() - click_info.x = data["points"][0]["x"] - click_info.y = data["points"][0]["y"] - click_info.idx = data["points"][0]["curveNumber"] - click_info.legend = figure["data"][click_info.idx]["name"] - - meta = figure["layout"].get("meta") - if isinstance(meta, list): - meta = meta[0] - - if not meta: - return "Error: no meta found for this graph ..." - if "name" not in meta: - return "Error: meta found for this graph has no name ..." - - obj = TableStats.stats_by_name[meta["name"]] - return obj.do_hover(meta.get("value"), variables, figure, data, click_info) - - @app.callback( - [Output("permalink", "href"), Output("download", "href")], - [Input(f"list-settings-{sanitize_setting_key(key)}", "value") for key in Matrix.settings] - + [ - Input("custom-config", "value"), - Input("custom-config-saved", "data-label"), - Input("settings-order", "data-order"), - ], - [State("custom-config-saved", "data-label")], - ) - def get_permalink_cb(*args): - try: - triggered_id = dash.callback_context.triggered - except IndexError: - return dash.no_update, dash.no_update # nothing triggered the script (on multiapp load) - - search = get_permalink(args) - - return search, "/matrix/dl" + search - - for _graph_idx, _graph_id in enumerate(GRAPH_IDS): - - def create_callback(graph_idx, graph_id): - @app.callback( - [Output(graph_id, "style"), Output(graph_id + "-txt", "style")], - [Input("list-settings-stats", "value")], - ) - def graph_style(stats_values): - try: - triggered_id = dash.callback_context.triggered[0]["prop_id"] - except IndexError: - triggered_id = None # nothing triggered the script (on multiapp load) - - if not isinstance(stats_values, list): - # only 1 elt in stats_values dropdown, a str is returned instead of a list. - # That makes the following silly ... - stats_values = [stats_values] - - if ( - graph_idx != "graph-for-dl" - and (graph_idx + 1) > len(stats_values) - or not triggered_id - or not (stats_values and stats_values[0]) - ): - return ( - {"display": "none"}, - {"display": "none"}, - ) - - graph_style = {} - - graph_style["display"] = "block" - graph_style["height"] = ( - f"{100 / (min(NB_GRAPHS, len(stats_values))) if stats_values else 100:.2f}vh" - ) - text_style = {"display": "block"} - - table_stat = TableStats.stats_by_name[stats_values[graph_idx]] - logging.info(f"Show {table_stat.name}") - - try: - if table_stat.no_graph: # may raise AttributeError - graph_style["display"] = "none" - except AttributeError: - pass - - return graph_style, text_style - - @app.callback( - [Output(graph_id, "figure"), Output(graph_id + "-txt", "children")], - [ - Input(f"list-settings-{sanitize_setting_key(key)}", "value") - for key in Matrix.settings - ] - + [Input("lbl_settings", "n_clicks")] - + [Input("settings-order", "data-order")] - + [ - Input("config-title", "n_clicks"), - Input("custom-config", "value"), - Input("custom-config-saved", "data-label"), - ], - [State("custom-config-saved", "data-label")], - ) - def graph_figure_cb(*args): - return graph_figure(*args) - - def graph_figure(*_args): - try: - triggered_id = dash.callback_context.triggered[0]["prop_id"] - except IndexError: - return dash.no_update, "" # nothing triggered the script (on multiapp load) - except dash.exceptions.MissingCallbackContextException: - triggered_id = "" - - *args, cfg_n_clicks, config, config_saved, config_init = _args - - if triggered_id == "custom-config.value": - if not config or config.startswith("_"): - return dash.no_update, dash.no_update - - class Cfg: - def __init__(self): - self.requests = set() - self.d = {} - - def get(self, key, default=...): - self.requests.add(key) - - return self.d[key] if default is ... else self.d.get(key, default) - - cfg = Cfg() - lst = ( - (config_saved if config_saved else []) - + (config_init if cfg_n_clicks is None else []) - + ([config] if config else []) - ) - - for cf in lst: - k, _, v = cf.partition("=") - if k.startswith("_"): - continue - v = int(v) if v.isdigit() else v - cfg.d[k] = v - - var_order = args[-1] - if not var_order: - var_order = list(Matrix.settings.keys()) - - settings = dict( - zip(Matrix.settings.keys(), args[: len(Matrix.settings)], strict=False) - ) - - stats_values = settings.get("stats") - if not stats_values: - return {}, "" - - if not isinstance(stats_values, list): - # only 1 elt in stats_values dropdown, a str is returned instead of a list. - # That makes the following silly ... - stats_values = [stats_values] - - if graph_idx != "graph-for-dl" and ( - not stats_values or (graph_idx + 1) > len(stats_values) - ): - return dash.no_update, dash.no_update - - try: - table_stat = TableStats.stats_by_name[stats_values[graph_idx]] - except KeyError: - return "Stat '{stats_values[graph_idx]}' not found ...", None - - ordered_vars = list({k for k, v in settings.items() if k != "stats" and v == "---"}) - ordered_vars.sort(key=lambda x: var_order.index(x) if x in var_order else 999) - - variables = {k: Matrix.settings[k] for k in ordered_vars} - - for k in list(variables.keys()): - if k.startswith("@"): - variables[k] = [v for v in variables[k] if v != ""] - - setting_lists = [[(key, v) for v in variables[key]] for key in ordered_vars] - - try: - plot_msg = table_stat.do_plot( - ordered_vars, settings, setting_lists, variables, cfg - ) - except Exception as e: - import bdb - - if isinstance(e, bdb.BdbQuit): - raise e - - msg = f"FAILED: {e.__class__.__name__}: {e}" - logging.error(msg) - - traceback.print_exc() - return None, msg - - if plot_msg is None: - msg = f"FAILED: {table_stat.do_plot.__qualname__} returned None ..." - logging.error(msg) - return None, msg - - plot, msg = plot_msg - - if "help" not in cfg.d: - return plot, msg - - # - # config help - # - - fig = go.Figure() - fig.update_layout(title="help: see below the graph ...") - - help_msg = [ - html.B(f"Config helper for '{stats_values[graph_idx]}' plot:"), - html.Ul([html.Li([html.I(k), " ⇒ ", cfg.d.get(k, "")]) for k in cfg.requests]), - ] - - return plot, help_msg - - if graph_id == "graph-for-dl": - TableStats.graph_figure = graph_figure - - # must use internal function to save 'table_stat' closure context - create_callback(_graph_idx, _graph_id) - create_callback(0, "graph-for-dl") diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/bWLwgP.css b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/bWLwgP.css deleted file mode 100644 index ae4e227e8..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/bWLwgP.css +++ /dev/null @@ -1,416 +0,0 @@ -// Downloaded from: https://codepen.io/chriddyp/pen/bWLwgP.css -/* Table of contents -–––––––––––––––––––––––––––––––––––––––––––––––––– -- Plotly.js -- Grid -- Base Styles -- Typography -- Links -- Buttons -- Forms -- Lists -- Code -- Tables -- Spacing -- Utilities -- Clearing -- Media Queries -*/ - -/* PLotly.js -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -/* plotly.js's modebar's z-index is 1001 by default - * https://github.com/plotly/plotly.js/blob/7e4d8ab164258f6bd48be56589dacd9bdd7fded2/src/css/_modebar.scss#L5 - * In case a dropdown is above the graph, the dropdown's options - * will be rendered below the modebar - * Increase the select option's z-index - */ - -/* This was actually not quite right - - dropdowns were overlapping each other (edited October 26) - -.Select { - z-index: 1002; -}*/ - - -/* Grid -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -.container { - position: relative; - width: 100%; - max-width: 960px; - margin: 0 auto; - padding: 0 20px; - box-sizing: border-box; } -.column, -.columns { - width: 100%; - float: left; - box-sizing: border-box; } - -/* For devices larger than 400px */ -@media (min-width: 400px) { - .container { - width: 85%; - padding: 0; } -} - -/* For devices larger than 550px */ -@media (min-width: 550px) { - .container { - width: 80%; } - .column, - .columns { - margin-left: 4%; } - .column:first-child, - .columns:first-child { - margin-left: 0; } - - .one.column, - .one.columns { width: 4.66666666667%; } - .two.columns { width: 13.3333333333%; } - .three.columns { width: 22%; } - .four.columns { width: 30.6666666667%; } - .five.columns { width: 39.3333333333%; } - .six.columns { width: 48%; } - .seven.columns { width: 56.6666666667%; } - .eight.columns { width: 65.3333333333%; } - .nine.columns { width: 74.0%; } - .ten.columns { width: 82.6666666667%; } - .eleven.columns { width: 91.3333333333%; } - .twelve.columns { width: 100%; margin-left: 0; } - - .one-third.column { width: 30.6666666667%; } - .two-thirds.column { width: 65.3333333333%; } - - .one-half.column { width: 48%; } - - /* Offsets */ - .offset-by-one.column, - .offset-by-one.columns { margin-left: 8.66666666667%; } - .offset-by-two.column, - .offset-by-two.columns { margin-left: 17.3333333333%; } - .offset-by-three.column, - .offset-by-three.columns { margin-left: 26%; } - .offset-by-four.column, - .offset-by-four.columns { margin-left: 34.6666666667%; } - .offset-by-five.column, - .offset-by-five.columns { margin-left: 43.3333333333%; } - .offset-by-six.column, - .offset-by-six.columns { margin-left: 52%; } - .offset-by-seven.column, - .offset-by-seven.columns { margin-left: 60.6666666667%; } - .offset-by-eight.column, - .offset-by-eight.columns { margin-left: 69.3333333333%; } - .offset-by-nine.column, - .offset-by-nine.columns { margin-left: 78.0%; } - .offset-by-ten.column, - .offset-by-ten.columns { margin-left: 86.6666666667%; } - .offset-by-eleven.column, - .offset-by-eleven.columns { margin-left: 95.3333333333%; } - - .offset-by-one-third.column, - .offset-by-one-third.columns { margin-left: 34.6666666667%; } - .offset-by-two-thirds.column, - .offset-by-two-thirds.columns { margin-left: 69.3333333333%; } - - .offset-by-one-half.column, - .offset-by-one-half.columns { margin-left: 52%; } - -} - - -/* Base Styles -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -/* NOTE -html is set to 62.5% so that all the REM measurements throughout Skeleton -are based on 10px sizing. So basically 1.5rem = 15px :) */ -html { - font-size: 62.5%; } -body { - font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */ - line-height: 1.6; - font-weight: 400; - font-family: "Open Sans", "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; - color: rgb(50, 50, 50); } - - -/* Typography -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -h1, h2, h3, h4, h5, h6 { - margin-top: 0; - margin-bottom: 0; - font-weight: 300; } -h1 { font-size: 4.5rem; line-height: 1.2; letter-spacing: -.1rem; margin-bottom: 2rem; } -h2 { font-size: 3.6rem; line-height: 1.25; letter-spacing: -.1rem; margin-bottom: 1.8rem; margin-top: 1.8rem;} -h3 { font-size: 3.0rem; line-height: 1.3; letter-spacing: -.1rem; margin-bottom: 1.5rem; margin-top: 1.5rem;} -h4 { font-size: 2.6rem; line-height: 1.35; letter-spacing: -.08rem; margin-bottom: 1.2rem; margin-top: 1.2rem;} -h5 { font-size: 2.2rem; line-height: 1.5; letter-spacing: -.05rem; margin-bottom: 0.6rem; margin-top: 0.6rem;} -h6 { font-size: 2.0rem; line-height: 1.6; letter-spacing: 0; margin-bottom: 0.75rem; margin-top: 0.75rem;} - -p { - margin-top: 0; } - - -/* Blockquotes -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -blockquote { - border-left: 4px lightgrey solid; - padding-left: 1rem; - margin-top: 2rem; - margin-bottom: 2rem; - margin-left: 0rem; -} - - -/* Links -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -a { - color: #1EAEDB; - text-decoration: underline; - cursor: pointer;} -a:hover { - color: #0FA0CE; } - - -/* Buttons -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -.button, -button, -input[type="submit"], -input[type="reset"], -input[type="button"] { - display: inline-block; - height: 38px; - padding: 0 30px; - color: #555; - text-align: center; - font-size: 11px; - font-weight: 600; - line-height: 38px; - letter-spacing: .1rem; - text-transform: uppercase; - text-decoration: none; - white-space: nowrap; - background-color: transparent; - border-radius: 4px; - border: 1px solid #bbb; - cursor: pointer; - box-sizing: border-box; } -.button:hover, -button:hover, -input[type="submit"]:hover, -input[type="reset"]:hover, -input[type="button"]:hover, -.button:focus, -button:focus, -input[type="submit"]:focus, -input[type="reset"]:focus, -input[type="button"]:focus { - color: #333; - border-color: #888; - outline: 0; } -.button.button-primary, -button.button-primary, -input[type="submit"].button-primary, -input[type="reset"].button-primary, -input[type="button"].button-primary { - color: #FFF; - background-color: #33C3F0; - border-color: #33C3F0; } -.button.button-primary:hover, -button.button-primary:hover, -input[type="submit"].button-primary:hover, -input[type="reset"].button-primary:hover, -input[type="button"].button-primary:hover, -.button.button-primary:focus, -button.button-primary:focus, -input[type="submit"].button-primary:focus, -input[type="reset"].button-primary:focus, -input[type="button"].button-primary:focus { - color: #FFF; - background-color: #1EAEDB; - border-color: #1EAEDB; } - - -/* Forms -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -input[type="email"], -input[type="number"], -input[type="search"], -input[type="text"], -input[type="tel"], -input[type="url"], -input[type="password"], -textarea, -select { - height: 38px; - padding: 6px 10px; /* The 6px vertically centers text on FF, ignored by Webkit */ - background-color: #fff; - border: 1px solid #D1D1D1; - border-radius: 4px; - box-shadow: none; - box-sizing: border-box; - font-family: inherit; - font-size: inherit; /*https://stackoverflow.com/questions/6080413/why-doesnt-input-inherit-the-font-from-body*/} -/* Removes awkward default styles on some inputs for iOS */ -input[type="email"], -input[type="number"], -input[type="search"], -input[type="text"], -input[type="tel"], -input[type="url"], -input[type="password"], -textarea { - -webkit-appearance: none; - -moz-appearance: none; - appearance: none; } -textarea { - min-height: 65px; - padding-top: 6px; - padding-bottom: 6px; } -input[type="email"]:focus, -input[type="number"]:focus, -input[type="search"]:focus, -input[type="text"]:focus, -input[type="tel"]:focus, -input[type="url"]:focus, -input[type="password"]:focus, -textarea:focus, -select:focus { - border: 1px solid #33C3F0; - outline: 0; } -label, -legend { - display: block; - margin-bottom: 0px; } -fieldset { - padding: 0; - border-width: 0; } -input[type="checkbox"], -input[type="radio"] { - display: inline; } -label > .label-body { - display: inline-block; - margin-left: .5rem; - font-weight: normal; } - - -/* Lists -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -ul { - list-style: circle inside; } -ol { - list-style: decimal inside; } -ol, ul { - padding-left: 0; - margin-top: 0; } -ul ul, -ul ol, -ol ol, -ol ul { - margin: 1.5rem 0 1.5rem 3rem; - font-size: 90%; } -li { - margin-bottom: 1rem; } - - -/* Tables -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -table { - border-collapse: collapse; -} -th:not(.CalendarDay), -td:not(.CalendarDay) { - padding: 12px 15px; - text-align: left; - border-bottom: 1px solid #E1E1E1; } -th:first-child:not(.CalendarDay), -td:first-child:not(.CalendarDay) { - padding-left: 0; } -th:last-child:not(.CalendarDay), -td:last-child:not(.CalendarDay) { - padding-right: 0; } - - -/* Spacing -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -button, -.button { - margin-bottom: 0rem; } -input, -textarea, -select, -fieldset { - margin-bottom: 0rem; } -pre, -dl, -figure, -table, -form { - margin-bottom: 0rem; } -p, -ul, -ol { - margin-bottom: 0.75rem; } - -/* Utilities -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -.u-full-width { - width: 100%; - box-sizing: border-box; } -.u-max-full-width { - max-width: 100%; - box-sizing: border-box; } -.u-pull-right { - float: right; } -.u-pull-left { - float: left; } - - -/* Misc -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -hr { - margin-top: 3rem; - margin-bottom: 3.5rem; - border-width: 0; - border-top: 1px solid #E1E1E1; } - - -/* Clearing -–––––––––––––––––––––––––––––––––––––––––––––––––– */ - -/* Self Clearing Goodness */ -.container:after, -.row:after, -.u-cf { - content: ""; - display: table; - clear: both; } - - -/* Media Queries -–––––––––––––––––––––––––––––––––––––––––––––––––– */ -/* -Note: The best way to structure the use of media queries is to create the queries -near the relevant code. For example, if you wanted to change the styles for buttons -on small devices, paste the mobile query code up in the buttons section and style it -there. -*/ - - -/* Larger than mobile */ -@media (min-width: 400px) {} - -/* Larger than phablet (also point when grid becomes active) */ -@media (min-width: 550px) {} - -/* Larger than tablet */ -@media (min-width: 750px) {} - -/* Larger than desktop */ -@media (min-width: 1000px) {} - -/* Larger than Desktop HD */ -@media (min-width: 1200px) {} diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/resizing_script.js b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/resizing_script.js deleted file mode 100644 index 8c3649409..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/assets/resizing_script.js +++ /dev/null @@ -1,20 +0,0 @@ -if (!window.dash_clientside) { - window.dash_clientside = {}; -} - -window.dash_clientside.clientside = { - resize: function(value) { - setTimeout(function() { - window.dispatchEvent(new Event("resize")); - }, 500); - return null; - }, - - resize_graph: function(input_graph_style,) { - window.dispatchEvent(new Event("resize")); - setTimeout(function() { - window.dispatchEvent(new Event("resize")); - }, 500); - return null; - }, -}; diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/report.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/report.py deleted file mode 100644 index 1a3f5b03a..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/report.py +++ /dev/null @@ -1,306 +0,0 @@ -import logging -import pathlib - -from dash import dcc - - -class _Report: - def __init__(self, id_name, index): - self.id_name = id_name - self.index = index - - self.figure_index = 0 - self.tabs_css_added = False - self.tabs_container_id = 0 - - def _children_element_to_html(self, elt): - props = " ".join( - [ - f"{k}='{getattr(elt, k)}'" - for k in elt.available_properties - if k not in ("children", "style") and hasattr(elt, k) - ] - ) - - if hasattr(elt, "style"): - if not isinstance(elt.style, dict): - logging.warning("The 'style' attribute should be a dict ...") - props += f" style='{elt.style}'" - else: - props += " style='" + " ".join([f"{k}:{v};" for k, v in elt.style.items()]) + "'" - - content = [ - f"<{elt._type.lower()}{' ' if props else ''}{props}{'/' if not elt.children else ''}>" - ] - - if elt.children is None: - return content - - if isinstance(elt.children, str): - content += [elt.children] - elif hasattr(elt.children, "_type"): - content += self._children_element_to_html(elt.children) - else: - try: - it = iter(elt.children) - except TypeError: - it = None # not iterable - - if it: - for child in it: - content += self._element_to_html(child) - else: - content += [str(elt.children)] - - content += [f""] - - return content - - def _graph_element_to_html(self, graph): - figure = graph.figure - if not figure: - return ["no graph available"] - - dirname = f"report_{self.index:02d}_{self.id_name}_files" - pathlib.Path(dirname).mkdir(exist_ok=True) - dest = f"{dirname}/fig_{self.figure_index}" - - self.figure_index += 1 - - logging.info(f"Saving {dest} ...") - dest_html = f"{dest}.html" - dest_png = f"{dest}.png" - - from .web import IMAGE_HEIGHT, IMAGE_WIDTH - - try: - figure.write_html(dest_html) - figure.write_image(dest_png, width=IMAGE_WIDTH, height=IMAGE_HEIGHT) - except Exception as e: - msg = f"Failed to save graph #{self.index} {self.id_name} '{figure.layout.title.text}':" - logging.exception(f"Failed to save graph #{self.index} {self.id_name}: {e}") - - return [f"

{msg}: {e}

"] - - return [ - f"

" - ] - - def _tabs_element_to_html(self, tabs): - """Convert dcc.Tabs to HTML with proper tab navigation""" - content = [] - - # Add CSS and JavaScript the first time tabs are used - if not self.tabs_css_added: - content.append(""" - - - -""") - self.tabs_css_added = True - - if not tabs.children: - return ["

No tabs available

"] - - # Generate unique container and tab IDs - container_id = f"tabs-container-{self.index}-{self.tabs_container_id}" - self.tabs_container_id += 1 - tab_count = len(tabs.children) - tab_ids = [f"tab-{self.index}-{self.tabs_container_id - 1}-{i}" for i in range(tab_count)] - - content.append(f"
") - - # Create tab headers - content.append("
") - for i, tab in enumerate(tabs.children): - label = getattr(tab, "label", f"Tab {i + 1}") - active_class = " active" if i == 0 else "" - content.append( - f"" - ) - content.append("
") - - # Create tab contents - for i, tab in enumerate(tabs.children): - active_class = " active" if i == 0 else "" - content.append(f"
") - if tab.children: - if isinstance(tab.children, list): - for child in tab.children: - content += self._element_to_html(child) - else: - content += self._element_to_html(tab.children) - content.append("
") - - content.append("
") - return content - - def _tab_element_to_html(self, tab): - """Convert individual dcc.Tab to HTML - used when tab is standalone""" - content = [] - - # Add tab label as heading if it exists - if hasattr(tab, "label") and tab.label: - content.append(f"

{tab.label}

") - - # Add tab content - if tab.children: - content.append("
") - if isinstance(tab.children, list): - for child in tab.children: - content += self._element_to_html(child) - else: - content += self._element_to_html(tab.children) - content.append("
") - - return content - - def _element_to_html(self, elt): - if elt is None: - return ["None"] - elif isinstance(elt, str): - return [elt] - elif any([isinstance(elt, t) for t in (str, float, int)]): - return [str(elt)] - elif isinstance(elt, dcc.Tabs): - return self._tabs_element_to_html(elt) - elif isinstance(elt, dcc.Tab): - return self._tab_element_to_html(elt) - elif hasattr(elt, "children"): - return self._children_element_to_html(elt) - elif isinstance(elt, dcc.Graph): - return self._graph_element_to_html(elt) - elif to_html := getattr(elt, "to_html", None): - return [to_html()] - else: - logging.warning("Unsupported report element: %s", elt.__class__.__name__) - return [str(elt)] - - def generate(self, content, report_index_f, include_header): - # HTML document structure with proper headers for emoji support - html_doc_start = [ - "", - "", - "", - " ", - " ", - f" {self.id_name.replace('_', ' ').title()}", - "", - "", - ] - - html_doc_end = ["", ""] - - html = [] - html += html_doc_start - - if include_header: - header = [ - "

Click on the image to open the interactive full-size view of the plot.
", - "In the interactive view, click in the legend to hide a line, double click to see only this line.

", - "

Back to the reports index.

", - ] - html += header - - html += self._element_to_html(content) - html += html_doc_end - - html_content = "\n".join(html) - - if self.index is not None: - dest = f"report_{self.index:02d}_{self.id_name}.html" - else: - dest = self.id_name - - logging.info(f"Saving {dest} ...") - with open(dest, "w", encoding="utf-8") as out_f: - print(html_content, file=out_f) - - if report_index_f is not None: - print( - f"
  • Report {self.index:02d}: {self.id_name.replace('_', ' ')}", - file=report_index_f, - ) - - -def generate(idx, id_name, content, report_index_f, include_header=True): - _Report(id_name, idx).generate(content, report_index_f, include_header) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/web.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/web.py deleted file mode 100644 index 26d725bae..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/plotting/ui/web.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -import os -import sys -import traceback - -import dash -import flask -from dash import dcc, html -from dash.dependencies import Input, Output - -try: - import numpy -except ImportError: - pass - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.plotting.ui as ui -import matrix_benchmarking.plotting.ui.report as report - -IMAGE_WIDTH = int(os.environ.get("MATBENCH_PLOTTING_IMAGE_WIDTH", 1200)) -IMAGE_HEIGHT = int(os.environ.get("MATBENCH_PLOTTING_IMAGE_HEIGHT", 650)) - -# stylesheets now served via assets/bWLwgP.css and automatically included -main_app = dash.Dash(__name__) - -log = logging.getLogger("werkzeug") -log.setLevel(logging.ERROR) - - -def construct_dispatcher(): - main_app.config.suppress_callback_exceptions = True - - main_app.layout = html.Div([dcc.Location(id="url", refresh=False), html.Div(id="page-content")]) - - @main_app.callback( - Output("page-content", "children"), [Input("url", "pathname"), Input("url", "search")] - ) - def display_page(pathname, search): - if pathname is None: - return - url = flask.request.referrer.split("/", maxsplit=3)[-1] # http://host/{key} - - if pathname.startswith("/viewer/"): - return "No viewer yet ..." - - elif pathname.startswith("/matrix"): - return ui.build_layout(search) - - elif pathname.startswith("/saved"): - return "No saved yet ..." - - else: - msg = f"Invalid page requested ({pathname})" if pathname != "/" else "" - - index = html.Ul( - [html.Li(html.A("Saved records index)", href="/viewer"))] - + [html.Li(html.A("Saved graphs index", href="/saved"))] - + ([html.Li(html.A("Matrix visualizer", href="/matrix"))]) - ) - - return [msg, index] - - -def run(): - ui.build_callbacks(main_app) - display_page = construct_dispatcher() - - generate = cli_args.kwargs["generate"] - - if generate: - logging.info(f"Generating http://127.0.0.1:8050/matrix?{generate.replace(' ', '%20')} ...") - - page = ui.build_layout(generate, serializing=True) - - idx = -1 - content = page.children[1].children - report_index_f = open("reports_index.html", "w") - print("
      ", file=report_index_f) - - for graph, text in zip(content[0::2], content[1::2], strict=False): - if not isinstance(graph, dcc.Graph): - continue - idx += 1 - - stats = table_stats.TableStats.stats_by_id[graph.id] - - if getattr(stats, "is_report", False): - report.generate(idx, graph.id, text, report_index_f) - continue - - figure = graph.figure - if not figure: - continue - - dest = f"{idx:02d}_{graph.id.replace(' ', '_').replace('/', '_')}" - - logging.info(f"Saving {dest} ...") - figure.write_html(f"fig_{dest}.html") - figure.write_image(f"fig_{dest}.png", width=IMAGE_WIDTH, height=IMAGE_HEIGHT) - - print("
    ", file=report_index_f) - report_index_f.close() - - sys.exit(0) - - try: - main_app.run() - except OSError as e: - if e.errno == 98: - logging.error("Dash server port already in use ...") - else: - logging.error(f"DASH: {e.__class__.__name__}: {e}") - traceback.print_exception(*sys.exc_info()) - except Exception as e: - logging.error(f"DASH: {e.__class__.__name__}: {e}") - traceback.print_exception(*sys.exc_info()) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/__init__.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/__init__.py deleted file mode 100644 index 955694fdc..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/__init__.py +++ /dev/null @@ -1,205 +0,0 @@ -import copy -import datetime -import importlib -import inspect -import logging -import pathlib -import sys -from collections import defaultdict - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.models as models -import matrix_benchmarking.store.simple as store_simple - - -def load_workload_store(kwargs): - workload = kwargs["workload"] - if workload is True: - raise ValueError("'workload' must have a value ...") - - if workload_base_dir := kwargs.get("workload_base_dir"): - logging.info( - f"Adding workload_base_dir='{workload_base_dir}' to the Python module path list" - ) - sys.path.insert(0, workload_base_dir) - - try: - module = f"{workload}.store" - logging.info(f"Loading {module} module ...") - - store_module = importlib.import_module(module) - except ModuleNotFoundError: - logging.error(f"Could not load '{module}' module :/") - raise - - for fct_name in ("parse_lts_data", "parse_data"): - if hasattr(store_module, fct_name): - continue - - setattr(store_module, fct_name, getattr(store_simple, fct_name)) - - logging.info(f"Loading {module} module ... done.") - - return store_module - - -def should_be_filtered_out(settings): - for key, _value in settings.items(): - if key not in cli_args.experiment_filters: - # Keep it - continue - value = str(_value) - filter_value = cli_args.experiment_filters[key] - if isinstance(filter_value, list): - if value in filter_value: - continue # Keep it - - elif filter_value == value: - continue # Keep it - - # Skip it - return True - - return False - - -def add_to_matrix( - import_settings, location, results, exit_code, duplicate_handler, matrix=common.Matrix -): - if should_be_filtered_out(import_settings): - return - - import_key = matrix.settings_to_key(import_settings) - - if import_key in matrix.import_map: - old_entry = matrix.import_map[import_key] - - old_location = getattr(old_entry, "location", None) - - duplicate_handler(import_key, old_entry, old_location, results, location) - - return - - is_lts = matrix != common.Matrix - try: - processed_settings = _rewrite_settings(dict(import_settings), results, is_lts) - except Exception as e: - logging.error(f"failed to rewrite settings for entry at '{location}'") - raise e - - if not processed_settings: - # logging.info(f"entry '{import_key}' skipped by rewrite_settings()") - matrix.import_map[import_key] = True, location - return - - if should_be_filtered_out(processed_settings): - return - - processed_key = matrix.settings_to_key(processed_settings) - - if processed_key in matrix.processed_map: - logging.warning(f"duplicated processed key: {processed_key}") - logging.warning(f"duplicated import key: {import_key}") - entry = matrix.processed_map[processed_key] - logging.warning(f" old: {entry.location}") - logging.warning(f" new: {location}") - matrix.import_map[import_key] = entry - - processed_settings["run"] = ( - str(processed_settings.get("run")) + "_" + datetime.datetime.now().strftime("%H%M%S.%f") - ) - processed_key = matrix.settings_to_key(processed_settings) - return - - entry = common.MatrixEntry( - location, - results, - exit_code, - processed_key, - import_key, - processed_settings, - import_settings, - matrix=matrix, - ) - - gather_rolling_entries(entry, matrix=matrix) - - return entry - - -def gather_rolling_entries(entry, matrix=common.Matrix): - gathered_settings = dict(entry.settings.__dict__) - gathered_keys = [] - for k in gathered_settings.keys(): - if not k.startswith("@"): - continue - gathered_settings[k] = "" - gathered_keys.append(k) - - if not gathered_keys: - return - - gathered_entry = matrix.get_record(gathered_settings) - if not gathered_entry: - processed_key = matrix.settings_to_key(gathered_settings) - import_key = None - import_settings = None - location = entry.location - gathered_entry = common.MatrixEntry( - location, - [], - None, - processed_key, - import_key, - gathered_settings, - import_settings, - matrix=matrix, - ) - gathered_entry.is_gathered = True - gathered_entry.gathered_keys = defaultdict(set) - - gathered_entry.results.append(entry) - for gathered_key in gathered_keys: - gathered_entry.gathered_keys[gathered_key].add(entry.settings.__dict__[gathered_key]) - - -# --- - -custom_rewrite_settings = None - - -def _rewrite_settings(import_settings, results, is_lts): - if custom_rewrite_settings is None: - logging.warning("No rewrite_setting function registered.") - return import_settings - - if "results" not in inspect.signature(custom_rewrite_settings).parameters: - return custom_rewrite_settings(import_settings) - - return custom_rewrite_settings(import_settings, results, is_lts) - - -def register_custom_rewrite_settings(fn): - global custom_rewrite_settings - custom_rewrite_settings = fn - - -# -- - -lts_schema = None - - -def register_lts_schema(schema): - global lts_schema - lts_schema = schema - - -def get_lts_schema(): - if lts_schema is None: - logging.warning("No LTS schema registered") - - return lts_schema - - -# --- diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/prom_db.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/prom_db.py deleted file mode 100755 index c2ade721a..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/prom_db.py +++ /dev/null @@ -1,247 +0,0 @@ -#! /usr/bin/env python3 - -import datetime -import json -import logging -import os -import pathlib -import shutil -import subprocess -import sys -import tarfile -import tempfile -import time - -PROMETHEUS_URL = "http://localhost:9090" - - -def _parse_metric_values_from_file(metric_file): - with open(metric_file) as f: - json_metrics = json.load(f) - - import matrix_benchmarking.models # import here, otherwise this file cannot be executed standalone - import pydantic - - return pydantic.parse_obj_as(matrix_benchmarking.models.PrometheusValues, json_metrics) - - -def _extract_metrics_from_prometheus(tsdb_path, process_metrics): - import prometheus_api_client # lazy loading ... - - metrics_values = {} - logging.info("Checking Prometheus availability ...") - - # ensure that prometheus is available - subprocess.run( - ["prometheus", "--help"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ) - - failed = False - try: - prom_cfg = os.environ.get("PROMETHEUS_CONFIG_FILE", "/etc/prometheus/prometheus.yml") - - prom_cmd = ["prometheus", "--storage.tsdb.path", str(tsdb_path), "--config.file", prom_cfg] - - prom_proc = subprocess.Popen(prom_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - - logging.info("Waiting for Prometheus to respond to its API ...") - RETRY_COUNT = 120 # 10 minutes - time.sleep(5) - for i in range(RETRY_COUNT): - if prom_proc.poll() is not None: - logging.error(f"Prometheus failed. Return code: {prom_proc.returncode}.") - failed = True - sys.exit(1) - - try: - prom_connect = prometheus_api_client.PrometheusConnect( - url=PROMETHEUS_URL, - disable_ssl=True, - ) - all_metrics = prom_connect.all_metrics() - break - except Exception: - logging.info(f"Not available ... {i}/{RETRY_COUNT}") - else: - logging.error("Could not connect to Promtheus, aborting.") - failed = True - sys.exit(1) - - process_metrics(prom_connect) - - print("Reading ... done") - finally: - logging.info("Terminating Prometheus ...") - prom_proc.terminate() - prom_proc.kill() - prom_proc.wait() - if failed or prom_proc.returncode: - logging.info( - "\n%s\n", - prom_proc.stderr.read().decode("utf8").strip(), - ) - - -def prepare_prom_db(prometheus_tgz, process_metrics): - logging.info(f"Processing {prometheus_tgz} ...") - if not tarfile.is_tarfile(prometheus_tgz): - logging.error(f"{prometheus_tgz} isn't a valid tar file.") - return - - try: - prom_db_tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="prometheus_db_")) - with tarfile.open(prometheus_tgz, "r:gz") as prometheus_tarfile: - prometheus_tarfile.extractall(prom_db_tmp_dir) - - _extract_metrics_from_prometheus(prom_db_tmp_dir, process_metrics) - except EOFError as e: - logging.error(f"File '{prometheus_tgz}' is an invalid tarball: %s", e) - except KeyboardInterrupt: - print("\n") - logging.error("Interrupted :/") - sys.exit(1) - finally: - shutil.rmtree(prom_db_tmp_dir, ignore_errors=True) - - -def extract_metrics(prometheus_tgz, metrics, dirname): - metric_results = {} - missing_metrics = [] - metrics_base_dir = dirname / "metrics" - - if not metrics: - return - - with open(dirname / "metrics.txt", "w") as f: - for metric in metrics: - metric_name, metric_query = ( - list(metric.items())[0] if isinstance(metric, dict) else (metric, metric) - ) - print(f"# {metric_name}", file=f) - print(f"{metric_query}", file=f) - - metric_filename = metric_name.replace(".*", "").replace("'", "").replace("~", "") - metric_file = metrics_base_dir / f"{metric_filename}.json" - if not metric_file.exists(): - missing_metrics.append([metric_name, metric_query, metric_file]) - logging.info(f"No cache available for metric '{metric_name}'") - continue - - metric_results[metric_name] = _parse_metric_values_from_file(metric_file) - - if not missing_metrics: - logging.debug("All the metrics files exist, no need to launch Prometheus.") - return metric_results - - metrics_base_dir.mkdir(exist_ok=True) - - metrics_values = {} - - def process_metrics(prom_connect): - nonlocal metrics_values - import prometheus_api_client.exceptions # lazy loading ... - - up_query = prom_connect.custom_query(query="up[60y]") - if not up_query: - logging.error( - f"No 'up' metric available in the database at '{prometheus_tgz}'. Cannot proceed :/" - ) - return - - all_start_times = [] - all_end_times = [] - - for target in up_query: - if not target["values"]: - continue - all_start_times.append(target["values"][0][0]) - all_end_times.append(target["values"][-1][0]) - - start_date = datetime.datetime.fromtimestamp(min(all_start_times)) - end_date = datetime.datetime.fromtimestamp(max(all_end_times)) - - del up_query # no need to keep it in memory - - duration = end_date - start_date - SECOND_PER_STEP = 300 - MIN_STEP = 5 - step = max(MIN_STEP, int(duration.total_seconds() / SECOND_PER_STEP)) - logging.info(f"Prometheus up time is {duration}. Using a step value of {step}.") - for metric_name, metric_query, metric_file in missing_metrics: - if "(" in metric_query: - try: - values = prom_connect.custom_query_range( - query=metric_query, step=step, start_time=start_date, end_time=end_date - ) - except prometheus_api_client.exceptions.PrometheusApiClientException as e: - logging.warning(f"Fetching {metric_query} raised an exception") - logging.warning(f"Exception: {e}") - continue - - metrics_values[metric_name] = metric_values = [] - if not values: - continue - # deduplicate the values - for current_values in values: - current_metric_values = {} - metric_values.append(current_metric_values) - current_metric_values["metric"] = current_values["metric"] # empty :/ - current_metric_values["values"] = [] - prev_val = None - prev_ts = None - has_skipped = False - for ts, val in current_values["values"]: - if val == prev_val: - has_skipped = True - prev_ts = ts - continue - if has_skipped: - current_metric_values["values"].append([prev_ts, prev_val]) - has_skipped = False - current_metric_values["values"].append([ts, val]) - prev_ts = ts - prev_val = val - if prev_val is not None and has_skipped: - # add the last value if the list wasn't empty - current_metric_values["values"].append([ts, val]) - - else: - metric_values = metrics_values[metric_name] = prom_connect.custom_query( - query=f"{metric_query}[60y]" - ) - - if not metric_values: - logging.warning(f"{metric_name} has no data :/") - - logging.info( - "Launching Prometheus instance to grab %s", - ", ".join([name for name, query, file in missing_metrics]), - ) - prepare_prom_db(prometheus_tgz, process_metrics) - - for metric_name, metric_query, metric_file in missing_metrics: - with open(metric_file, "w") as f: - json.dump(metrics_values.get(metric_name, []), f) - - logging.info(f"Metric {metric_name} fetched and stored.") - metric_results[metric_name] = _parse_metric_values_from_file(metric_file) - - return metric_results - - -if __name__ == "__main__": - logging.basicConfig( - level=os.environ.get("LOGLEVEL", "INFO"), - format="%(levelname)s | %(message)s", - ) - - def process_metrics(prom_connect): - print("Prometheus is listing on", PROMETHEUS_URL) - msg = input("Press enter to terminate it. (or type 'pdb' to enter pdb debugger) ") - if msg == "pdb": - import pdb - - pdb.set_trace() - pass - - prepare_prom_db(sys.argv[1], process_metrics) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/simple.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/simple.py deleted file mode 100644 index a4b9a8e53..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/store/simple.py +++ /dev/null @@ -1,290 +0,0 @@ -import json -import logging -import os -import pathlib -import shutil -import types - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.store as store -import yaml -from matrix_benchmarking import download_lts - - -def invalid_directory(dirname, settings, reason, warn=False): - run_flag = cli_args.kwargs.get("run") - clean_flag = cli_args.kwargs.get("clean") - benchmark_mode = cli_args.kwargs["execution_mode"] - - if warn or clean_flag: - logging.info("%s", dirname) - logging.info("%s", ", ".join(f"{k}={v}" for k, v in settings.items())) - logging.info("\t\tis invalid: %s", reason) - logging.info("") - - if benchmark_mode: - return - if not run_flag: - return - if not clean_flag: - return - - shutil.rmtree(dirname) - logging.info(f"{dirname}: removed ({reason})") - - -def _duplicated_directory(import_key, old_entry, old_location, new_results, new_location): - logging.warning(f"duplicated results key: {import_key}") - logging.warning(f" old: {old_location}") - logging.warning(f" new: {new_location}") - - if not cli_args.kwargs.get("clean"): - return - - if not cli_args.kwargs["run"]: - logging.info(f"{new_location} would have been deleted.") - return - - shutil.rmtree(new_location) - logging.info(f"{new_location}: removed") - - -def parse_old_settings(filename): - settings = {} - with open(filename) as f: - for line in f.readlines(): - if not line.strip(): - continue - - key, found, value = line.strip().partition("=") - if not found: - logging.error( - f"Cannot parse setting from {filename}: invalid line (no '='): {line.strip()}" - ) - continue - - settings[key] = value - - return settings - - -def parse_settings(dirname): - import_settings = {} - - # search for settings[.*] in dirname and all of its parent directories. - # start in the top-most parent, so that each subdirectory overrides its parents. - for parent_dir in list(reversed([dirname] + list(dirname.parents))): - for filename in list(parent_dir.glob("settings")) + list(parent_dir.glob("settings.*")): - if filename.suffix not in (".yaml", ".yml"): # deprecated - logging.debug(f"Found deprecated 'settings' file in {dirname}: {filename}") - - import_settings.update(parse_old_settings(filename)) - continue - with open(filename) as f: - settings = yaml.safe_load(f) - import_settings.update(settings) - - return import_settings - - -def _parse_directory(results_dir, dirname): - import_settings = parse_settings(dirname) - - if store.should_be_filtered_out(import_settings): - return - - exit_code = -1 - try: - with open(dirname / "exit_code") as f: - content = f.read().strip() - if not content: - logging.info(f"{dirname}: exit_code is empty, skipping ...") - return - - exit_code = int(content) - except FileNotFoundError: - invalid_directory(dirname, import_settings, "exit_code not found") - return - - except Exception as e: - logging.info(f"{dirname}: exit_code cannot be read/parsed, skipping ... ({e})") - return - - def add_to_matrix(results, extra_settings=None): - if extra_settings: - entry_import_settings = dict(import_settings) - entry_import_settings.update(extra_settings) - else: - entry_import_settings = import_settings - - store.add_to_matrix( - entry_import_settings, pathlib.Path(dirname), results, exit_code, _duplicated_directory - ) - - try: - extra_settings__results = _parse_results(add_to_matrix, dirname, import_settings, exit_code) - except Exception as e: - logging.error(f"Failed to parse {dirname} ...") - logging.info(f" {e.__class__.__name__}: {e}") - logging.info("") - raise e - - -# --- - -custom_parse_results = None -custom_build_lts_payloads = None - - -def _parse_results(add_to_matrix, dirname, import_settings, exit_code): - if custom_parse_results is None: - raise RuntimeError("simple store: No data parser registered :/") - - return custom_parse_results(add_to_matrix, dirname, import_settings, exit_code) - - -def build_lts_payloads(): - if custom_build_lts_payloads is None: - raise RuntimeError("simple store: No payload builder registered :/") - - return custom_build_lts_payloads() - - -def register_custom_parse_results(fn): - global custom_parse_results - custom_parse_results = fn - - -def register_custom_build_lts_payloads(fn): - global custom_build_lts_payloads - custom_build_lts_payloads = fn - - -# --- -# Borrowed from: -# https://dev.to/taqkarim/extending-simplenamespace-for-nested-dictionaries-58e8 - - -# Similar to SimpleNamespace, but RecursiveNamespace.map_entry recursively creates namespaces from nested dicts -class RecursiveNamespace(types.SimpleNamespace): - @staticmethod - def map_entry(entry): - if isinstance(entry, dict): - return RecursiveNamespace(**entry) - - return entry - - def __init__(self, **kwargs): - super().__init__(**kwargs) - for key, val in kwargs.items(): - if type(val) == dict: - setattr(self, key, RecursiveNamespace(**val)) - elif type(val) == list: - setattr(self, key, list(map(self.map_entry, val))) - - -def parse_lts_data(lts_results_dir=None): - if lts_results_dir is None: - lts_results_dir = pathlib.Path(cli_args.kwargs["lts_results_dirname"]) - - def has_lts_anchor(files): - return download_lts.LTS_ANCHOR_NAME in files - - path = os.walk(lts_results_dir, followlinks=True) - for _this_dir, directories, files in path: - this_dir = pathlib.Path(_this_dir) - if "skip" in files: - continue - if not has_lts_anchor(files): - continue - - with open(this_dir / download_lts.LTS_ANCHOR_NAME) as f: - lts_anchor = yaml.safe_load(f) - - for filename in files: - if filename == download_lts.LTS_ANCHOR_NAME: - continue - if filename.startswith("."): - continue - - filepath = this_dir / filename - with open(filepath) as f: - document = json.load(f) - - lts_payload = RecursiveNamespace.map_entry(document) - - lts_settings = lts_payload.metadata.settings - - import_settings = dict(lts_settings.__dict__) - - import_settings["@timestamp"] = str(lts_payload.metadata.start) - - exit_code = getattr(lts_payload.metadata, "exit_code", None) - - def _duplicated_entry(import_key, old_entry, old_location, new_results, new_location): - logging.warning(f"duplicated results key: {import_key}") - - logging.warning(f" old: {old_location} | {old_entry.results.metadata.test_uuid}") - logging.warning(f" new: {new_location} | {new_results.metadata.test_uuid}") - - store.add_to_matrix( - import_settings, - filepath, - lts_payload, - exit_code, - _duplicated_entry, - matrix=common.LTS_Matrix, - ) - pass - - -# --- - - -def parse_data(results_dir=None): - if results_dir is None: - results_dir = pathlib.Path(cli_args.kwargs["results_dirname"]) - - if not results_dir.exists(): - raise FileNotFoundError(f"Results directory '{results_dir}' does not exist.") - - if not results_dir.is_dir(): - raise FileNotFoundError(f"Results directory '{results_dir}' is not a directory ...") - - def has_settings(files): - if "settings" in files: - logging.debug("Found deprecated 'settings' file ...") - return True # deprecated - if "settings.yml" in files: - logging.warning("Found settings file with invalid extention 'settings.yml' file ...") - return True - - if "settings.yaml" in files: - return True - - return False - - results_directories = [] - path = os.walk(results_dir, followlinks=True) - for _this_dir, directories, files in path: - if "skip" in files: - continue - if not has_settings(files): - continue - - this_dir = pathlib.Path(_this_dir) - - is_subdir_of_results_dir = False - for existing_results_directory in results_directories: - if existing_results_directory in this_dir.parents: - is_subdir_of_results_dir = True - break - if is_subdir_of_results_dir: - # we don't want nested results dirs - continue - - relative = this_dir.relative_to(results_dir) - - results_directories.append(this_dir) - _parse_directory(results_dir, this_dir) diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/upload_lts.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/upload_lts.py deleted file mode 100644 index 3fa42101e..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/upload_lts.py +++ /dev/null @@ -1,253 +0,0 @@ -import functools -import json -import logging - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.download_lts as download_lts -import matrix_benchmarking.generate_lts_schema as generate_lts_schema -import matrix_benchmarking.parse as parse -import matrix_benchmarking.store as store - - -def main( - workload: str = "", - workload_base_dir: str = "", - results_dirname: str = "", - opensearch_host: str = "", - opensearch_port: str = "", - opensearch_username: str = "", - opensearch_password: str = "", - opensearch_index: str = "", - filters: list[str] = [], - dry_run: bool = False, - upload_by_kpi: bool = False, -): - """ - Upload MatrixBenchmark LTS payloads to OpenSearch - - Upload MatrixBenchmark LTS payloads to OpenSearch, expects OpenSearch credentials and configuration to be available either in the enviornment or in an env file. - - Args: - workload: name of the workload to execute. (Mandatory) - workload_base_dir: the directory from where the workload packages should be loaded. (Optional) - - results_dirname: name of the directory where the results are stored. Can be set in the benchmark file. (Mandatory) - - opensearch_host: hostname of the OpenSearch instance - opensearch_port: port of the OpenSearch instance - opensearch_username: username of the OpenSearch instance - opensearch_password: password of the OpenSearch instance - opensearch_index: the OpenSearch index where the LTS payloads are stored (Mandatory) - - filters: If provided, parse and upload only the experiment matching the filters. Eg: expe=expe1:expe2,something=true. (Optional.) - dry_run: If provided, only parse results and not upload results to horreum. (Optional.) - upload_by_kpi: If enabled, upload the KPIs in a dedicated index (.) - """ - - kwargs = dict(locals()) # capture the function arguments - - optionals_flags = ["filters", "workload_base_dir", "dry_run", "upload_by_kpi"] - safe_flags = ["results_dirname", "workload", "opensearch_index"] + optionals_flags - - cli_args.setup_env_and_kwargs(kwargs) - cli_args.check_mandatory_kwargs( - kwargs, - mandatory_flags=[k for k in kwargs.keys() if k not in optionals_flags], - sensitive_flags=[k for k in kwargs.keys() if k not in safe_flags], - ) - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="upload-lts") - - workload_store = store.load_workload_store(kwargs) - - if kwargs.get("dry_run"): - logging.warning("Running in dry mode.") - - logging.info("Loading results ... ") - workload_store.parse_data() - logging.info(f"Loading results: done, found {len(common.Matrix.processed_map)} results") - - common.Matrix.print_settings_to_log() - common.Matrix.uniformize_settings_keys() - - client = ( - download_lts.connect_opensearch_client(kwargs) if not kwargs.get("dry_run") else None - ) - - logging.info(f"Uploading to OpenSearch /{kwargs.get('opensearch_index')}...") - - return upload( - client, - workload_store, - kwargs.get("dry_run"), - kwargs.get("opensearch_index"), - kwargs.get("upload_by_kpi"), - ) - - return cli_args.TaskRunner(run) - - -def opensearch_create_index(client, dry_run, opensearch_index): - if dry_run: - logging.info(f"Check if index '{opensearch_index}' exists or create it. (dry run)") - return - - index_body = { - "mappings": { - "properties": { - "metadata": {"type": "flat_object"}, - "kpis": {"type": "flat_object"}, - "results": {"type": "flat_object"}, - "regression_results": {"type": "flat_object"}, - }, - }, - "settings": { - "index": {"number_of_shards": 4}, - "mapping": { - "total_fields": { - "limit": 4000, - }, - }, - }, - } - - # Check if the index already exists - if client.indices.exists(index=opensearch_index): - del index_body["settings"]["index"][ - "number_of_shards" - ] # Can't update non dynamic settings [[index.number_of_shards]] for open indices - try: - client.indices.put_settings(index=opensearch_index, body=index_body) - except Exception as e: - logging.warn(f"Index '{opensearch_index}' could not be updated: {e}") - else: - logging.info(f"Index '{opensearch_index}' updated.") - - return - - # create the index - client.indices.create(index=opensearch_index, body=index_body) - logging.info(f"Index '{opensearch_index}' created.") - - -def update_mapping(client, workload_store, dry_run, opensearch_index): - schema = store.get_lts_schema() - mapping = generate_lts_schema.create_opensearch_mapping(schema.schema()) - - import pdb - - pdb.set_trace() - pass - - -def get_kpi_index_name(opensearch_index, kpi_name): - return f"{opensearch_index}.{kpi_name}" - - -def create_indexes(client, workload_store, dry_run, opensearch_index, lts_payloads, upload_by_kpi): - indexes_to_create = set() - indexes_to_create.add(opensearch_index) - - if upload_by_kpi: - for lts_payload, _, _ in lts_payloads: - payload_json = json.dumps( - lts_payload, default=functools.partial(parse.json_dumper, strict=False) - ) - payload_dict = json.loads(payload_json) - - for kpi_name, kpi in payload_dict.get("kpis", {}).items(): - kpi_index = get_kpi_index_name(opensearch_index, kpi_name) - indexes_to_create.add(kpi_index) - - logging.info(f"Creating/updating {len(indexes_to_create)} indexes ...") - for index_name in indexes_to_create: - opensearch_create_index(client, dry_run, index_name) - - -def upload(client, workload_store, dry_run, opensearch_index, upload_by_kpi): - variables = [k for k, v in common.Matrix.settings.items() if len(v) > 1] - - lts_payloads = list(workload_store.build_lts_payloads()) - - create_indexes(client, workload_store, dry_run, opensearch_index, lts_payloads, upload_by_kpi) - - for idx, (payload, start, end) in enumerate(lts_payloads): - try: - settings_dict = parse.json_dumper(payload.metadata.settings, strict=False) - key = ",".join( - f"{k}={v}" for k, v in settings_dict.items() if (not variables or k in variables) - ) - except Exception as e: - logging.warning(f"Failed to compute the name of the entry: {e}") - key = str(payload.metadata.settings) - - logging.info(f"Uploading payload #{idx} | {key}") - - payload_json = json.dumps( - payload, default=functools.partial(parse.json_dumper, strict=False) - ) - payload_dict = json.loads(payload_json) - - if upload_by_kpi: - upload_kpis_to_opensearch(client, payload_dict, dry_run, opensearch_index) - upload_lts_to_opensearch(client, payload_dict, dry_run, opensearch_index) - upload_regression_results_to_opensearch(client, payload_dict, dry_run, opensearch_index) - - logging.info("All done :)") - - -def upload_lts_to_opensearch(client, payload_dict, dry_run, opensearch_index): - logging.info(f"Uploading the LTS document to /{opensearch_index} ...") - - return upload_to_opensearch( - client, payload_dict, payload_dict["metadata"]["test_uuid"], dry_run, opensearch_index - ) - - -def upload_kpis_to_opensearch(client, payload_dict, dry_run, opensearch_index): - if "kpis" not in payload_dict.keys(): - logging.info("==> no KPI found in the payload.") - return - - for kpi_name, kpi in payload_dict.get("kpis", {}).items(): - kpi_index = get_kpi_index_name(opensearch_index, kpi_name) - logging.info(f"Uploading the KPI to /{kpi_index} ...") - - upload_to_opensearch(client, kpi, kpi["test_uuid"], dry_run, kpi_index) - - -def upload_regression_results_to_opensearch(client, payload_dict, dry_run, opensearch_index): - if not payload_dict.get("regression_results"): - logging.info("==> no regression results found in the payload.") - return - - logging.info("Uploading regression results to opensearch ... (stub)") - for idx, regression_result in enumerate(payload_dict["regression_results"]): - regression_result_index = f"{opensearch_index}__regression_results_{idx}" - opensearch_create_index(client, True, regression_result_index) - - logging.info(regression_result) - upload_to_opensearch( - client, - regression_result, - payload_dict["metadata"]["test_uuid"], - True, - regression_result_index, - ) - - -def upload_to_opensearch(client, document, document_id, dry_run, index): - if dry_run: - logging.info("==> skip upload (dry run)") - return - - response = client.index( - index=index, - body=document, - refresh=True, - id=document_id, - ) - - logging.info(f"==> {response['result']}") diff --git a/projects/matrix_benchmarking/subproject/matrix_benchmarking/visualize.py b/projects/matrix_benchmarking/subproject/matrix_benchmarking/visualize.py deleted file mode 100644 index 1b2559063..000000000 --- a/projects/matrix_benchmarking/subproject/matrix_benchmarking/visualize.py +++ /dev/null @@ -1,92 +0,0 @@ -import logging - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.common as common -import matrix_benchmarking.store as store - - -def main( - workload: str = "", - workload_base_dir: str = "", - results_dirname: str = "", - lts_results_dirname: str = "", - filters: list[str] = [], - generate: str = "", -): - """ - Visualize MatrixBenchmarking results. - - Env: - MATBENCH_WORKLOAD - MATBENCH_WORKLOAD_BASE_DIR - MATBENCH_RESULTS_DIRNAME - MATBENCH_LTS_RESULTS_DIRNAME - MATBENCH_GENERATE - MATBENCH_FILTERS - - See the `FLAGS` section for the descriptions. - - Args: - workload: Name of the workload to execute. (Mandatory.) - workload_base_directory: the directory from where the workload packages should be loaded. (Optional) - results_dirname: Name of the directory where the results will be stored. (Mandatory.) - lts_results_dirname: Name of the directory where the LTS results are stored. (Mandatory.) - generate: If set, the value is used as query to generates image files instead of running the Web UI. - filters: If provided, parse only the experiment matching the filters. Eg: expe=expe1:expe2,something=true. - lts: If 'True', invoke the LTS parser only. - """ - kwargs = dict(locals()) # capture the function arguments - - # lazy loading, to avoid importing these modules when not running in visualization mode - - import matrix_benchmarking.plotting.table_stats as table_stats - import matrix_benchmarking.plotting.ui as ui - import matrix_benchmarking.plotting.ui.web as ui_web - - cli_args.setup_env_and_kwargs(kwargs) - - cli_args.check_mandatory_kwargs(kwargs, ("workload", "results_dirname")) - - if generate and not isinstance(generate, str): - logging.error("The 'generate' flag must provide the query of the graph to generate.") - return 1 - - def run(): - cli_args.store_kwargs(kwargs, execution_mode="visualize") - - workload_store = store.load_workload_store(kwargs) - - # --- - - logging.info("Loading results ... ") - workload_store.parse_data() - logging.info(f"Loading results ... done. Found {len(common.Matrix.processed_map)} results.") - if not common.Matrix.processed_map: - logging.error("Not result found, exiting.") - return 1 - - common.Matrix.uniformize_settings_keys() - - common.Matrix.print_settings_to_log() - - if kwargs.get("lts_results_dirname"): - logging.info("--- LTS --- ") - workload_store.parse_lts_data() - common.LTS_Matrix.print_settings_to_log() - logging.info( - f"Loading LTS results ... done. Found {len(common.LTS_Matrix.processed_map)} results." - ) - - try: - ui.configure(kwargs, workload_store) - except Exception as e: - logging.error(f"Failed to configure the plotting module: {e}") - raise e - - table_stats.register_all() - - ui_web.run() - - return 0 - - return cli_args.TaskRunner(run) diff --git a/projects/matrix_benchmarking/subproject/requirements-exec.txt b/projects/matrix_benchmarking/subproject/requirements-exec.txt deleted file mode 100644 index 10a6b4de4..000000000 --- a/projects/matrix_benchmarking/subproject/requirements-exec.txt +++ /dev/null @@ -1,3 +0,0 @@ -prometheus-api-client -kubernetes -jinja2 diff --git a/projects/matrix_benchmarking/subproject/requirements.txt b/projects/matrix_benchmarking/subproject/requirements.txt deleted file mode 100644 index b8d32bdd4..000000000 --- a/projects/matrix_benchmarking/subproject/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -pandas -dash -plotly -kaleido==0.2.1 # version 0.4.1 crashes, it seems to require a browser (relies on choreographer) -fire -pyyaml -BeautifulSoup4==4.11.* -prometheus_api_client -pydantic==1.10.* -opensearch-py -boto3 -pandas -git+https://git@github.com/openshift-psap/hunter@b7ef087a0279554a529e440951f3d6096f9eacf3 # stream-10 diff --git a/projects/matrix_benchmarking/subproject/utils/wdm.py b/projects/matrix_benchmarking/subproject/utils/wdm.py deleted file mode 100755 index 0fb1351a5..000000000 --- a/projects/matrix_benchmarking/subproject/utils/wdm.py +++ /dev/null @@ -1,174 +0,0 @@ -#! /usr/bin/python3 - -import os -import subprocess -import sys -import tempfile - -import yaml - -deps = {} -resolved = set() - -tested = set() -installed = set() - - -def run_ansible(task, depth): - tmp = tempfile.NamedTemporaryFile("w+", dir=os.getcwd(), delete=False) - - play = [ - dict( - name=f"Run {task['name']}", - connection="local", - gather_facts=False, - hosts="localhost", - tasks=task["spec"], - ) - ] - - yaml.dump(play, tmp) - tmp.close() - - print("-" * (depth + 2)) - env = os.environ.copy() - env["ANSIBLE_CONFIG"] = "/home/kevin/openshift/ci-artifacts/config/ansible.cfg" - try: - sys.stdout.flush() - sys.stderr.flush() - proc = subprocess.run(["/usr/bin/ansible-playbook", tmp.name], env=env, stdin=None) - - ret = proc.returncode - finally: - os.remove(tmp.name) - pass - print("-" * (depth + 2)) - - return ret == 0 - - pass - - -def run_shell(task, depth): - cmd = task["spec"] - print(" " * depth, f"|>SHELL<| \n{cmd.strip()}") - - print("-" * (depth + 2)) - sys.stdout.flush() - sys.stderr.flush() - proc = subprocess.run(["/bin/bash", "-c", cmd], stdin=subprocess.PIPE) - ret = proc.returncode - print("-" * (depth + 2)) - - return ret == 0 - - -def run(task, depth): - print(" " * depth, f"|Running '{task['name']}' ...") - type_ = task["type"] - if type_ == "shell": - success = run_shell(task, depth) - elif type_ == "ansible": - success = run_ansible(task, depth) - else: - print(f"ERROR: unknown task type: {type_}.") - sys.exit(1) - - print(" " * depth, f"|Running '{task['name']}':", "Success" if success else "Failure") - print(" " * depth, "|___") - return success - - -def do_test(dep, depth, print_first_test=True): - if not dep["spec"].get("tests"): - if print_first_test: - print(f"Nothing to test for '{dep['name']}'") - return True - - for task in dep["spec"].get("tests", []): - if print_first_test: - print(" " * depth, f"Testing '{dep['name']}' ...") - print_first_test = False - - tested.add(f"{dep['name']} -> {task['name']}") - success = run(task, depth) - if success: - return True - - return False - - -def resolve(dep, depth=0): - print(" " * depth, f"Treating '{dep['name']}' dependency ...") - - if dep["name"] in resolved: - print(" " * depth, f"Dependency '{dep['name']}' has already need resolved, skipping.") - return - - for req in dep["spec"].get("requirements", []): - print(" " * depth, f"Dependency '{dep['name']}' needs '{req}' ...") - try: - next_dep = deps[req] - except KeyError: - print(f"ERROR: missing dependency: {req}") - sys.exit(1) - resolve(next_dep, depth=depth + 1) - - print(" " * depth, f"Nothing to test for '{dep['name']}'.") - - if do_test(dep, depth): - print(" " * depth, f"Dependency '{dep['name']}' is satisfied, no need to install.") - - else: - first_install = True - for task in dep["spec"].get("install", []): - if first_install: - first_install = False - print(" " * depth, f"Installing '{dep['name']}' ...") - - if not run(task, depth): - print(f"ERROR: install of '{dep['name']}' failed.") - sys.exit(1) - - installed.add(f"{dep['name']} -> {task['name']}") - - if first_install: - # no install task available - - print(f"ERROR: '{dep['name']}' test failed, but no install script provided.") - sys.exit(1) - - if not do_test(dep, depth, print_first_test=False): - print(f"ERROR: '{dep['name']}' installed, but test still failing.") - sys.exit(1) - - resolved.add(dep["name"]) - print(" " * depth, f"Done with {dep['name']}.") - - -def main(): - with open(sys.argv[1]) as f: - docs = list(yaml.safe_load_all(f)) - - for doc in docs: - deps[doc["name"]] = doc - - try: - entrypoint = sys.argv[2] - except IndexError: - entrypoint = docs[0]["name"] - - resolve(deps[entrypoint]) - print("All done.") - - print("Tested:") - [print(f"- {t}") for t in tested] - if installed: - print("Installed:") - [print(f"- {t}") for t in installed] - else: - print("Installed: nothing.") - - -if __name__ == "__main__": - main() diff --git a/projects/matrix_benchmarking/visualizations/helpers/analyze/__init__.py b/projects/matrix_benchmarking/visualizations/helpers/analyze/__init__.py deleted file mode 100644 index 1db289c24..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/analyze/__init__.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging - -import matrix_benchmarking.common as common -import pandas as pd - - -def prepare_regression_data( - comparison_keys, ignored_keys, rewrite_settings, sorting_keys=[], ignored_entries={} -): - logging.info("Preparing the regression analyses records ...") - - data = [] - for ref_entry in common.Matrix.all_records(): - lts_payload = ref_entry.results.lts - - if not hasattr(lts_payload, "kpis"): - logging.warning("No KPIs available ...") - continue - - kpis = lts_payload.kpis - - def get_name(settings): - return ( - getattr(settings, comparison_keys[0]) - if len(comparison_keys) == 1 - else "|".join([f"{k}={settings.__dict__[k]}" for k in comparison_keys]) - ) - - kpi_settings = ref_entry.results.lts.metadata.settings - ref_name = get_name(kpi_settings) - - ignore_this_entry = False - - for ignored_key, ignored_values in ignored_entries.items(): - ignored_key_value = kpi_settings.__dict__.get(ignored_key, None) - # ignored key not part of this entry. Keep it. - if ignored_key_value is None: - continue - - if isinstance(ignored_values, list): - if ignored_key_value in ignored_values: - ignore_this_entry = True - elif ignored_key_value == ignored_values: - ignore_this_entry = True - - if ignore_this_entry: - break - - if ignore_this_entry: - continue - - current_row = { - ref_name: ref_entry, - "ref": ref_name, - } - - for entry in common.LTS_Matrix.similar_records( - kpi_settings, - ignore_keys=(comparison_keys + ignored_keys), - rewrite_settings=rewrite_settings, - gathered=True, - ): - name = get_name(entry.settings) - if name == ref_name: - continue - current_row[name] = entry - - data.append(current_row) - - return pd.DataFrame(data), comparison_keys, ignored_keys, sorting_keys diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/html.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/html.py deleted file mode 100644 index e2681e1ba..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/html.py +++ /dev/null @@ -1,63 +0,0 @@ -from dash import html - -from . import report -from . import styles_css as css - - -def info_card(title, items): - """Create an information card with a list of key-value items. - - Args: - title (str): The title of the information card. - items (list): A list of tuples containing key-value pairs and their metadata. - - Returns: - dash.html.Div: The HTML representation of the information card. - """ - - item_elements = [] - for key, value, is_last, is_highlight in items: - value_element = value - if is_highlight and isinstance(value, str): - value_element = html.Span(value, style=css.STYLE_INFO_VALUE_HIGHLIGHT) - - item_style = css.STYLE_INFO_ITEM_LAST if is_last else css.STYLE_INFO_ITEM - item_elements.append( - html.Div( - [ - html.Div(f"{key}:", style=css.STYLE_INFO_KEY), - html.Div(value_element, style=css.STYLE_INFO_VALUE), - ], - style=item_style, - ) - ) - - return html.Div( - [ - html.Div(title, style=css.STYLE_INFO_CARD_HEADER), - html.Div(item_elements, style=css.STYLE_INFO_CARD_BODY), - ], - style=css.STYLE_INFO_CARD, - ) - - -def plot_card(plot_name, config): - """Create a plot card with proper HTML structure. - - Args: - plot_name (str): The name of the plot. - config (dict): The configuration settings for the plot. - - Returns: - dash.html.Div: The HTML representation of the plot card. - """ - - return html.Div( - [ - html.Div( - html.Div(report.Plot_and_Text(plot_name, config), className="plot-container"), - style=css.STYLE_PLOT_CONTENT, - ) - ], - style=css.STYLE_PLOT_CARD, - ) diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_plot.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_plot.py deleted file mode 100644 index 0c8341fb0..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_plot.py +++ /dev/null @@ -1,219 +0,0 @@ -import copy -import numbers - -import matrix_benchmarking.analyze.report as analyze_report -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.table_stats as table_stats -import pandas as pd -import plotly.express as px -from dash import html - -import projects.matrix_benchmarking.visualizations.helpers.plotting.report as report - - -def register(): - KpiPlot() - KpiPlotReport() - - -def generateData(entries, x_key, kpi_name, _variables, filter_key=None, filter_value=None): - data = [] - - variables = dict(_variables) - properties = None - - if x_key in variables: - del variables[x_key] - - for entry in entries: - if filter_key is not None and entry.get_settings()[filter_key] != filter_value: - continue - - datum = dict() - datum[x_key] = entry.settings.__dict__[x_key] if x_key else None - - kpi = entry.results.lts.kpis[kpi_name] - if not properties: - properties = dict( - unit=kpi.divisor_unit if "divisor" in kpi.__dict__ else kpi.unit, - title=kpi.help, - type=kpi.unit, - ) - - value = kpi.value - if kpi.__dict__.get("divisor"): - if isinstance(value, list): - value = [v / kpi.divisor for v in value] - else: - value /= kpi.divisor - - text = analyze_report.format_kpi_value(kpi) - - datum["y"] = value - datum["text"] = text - datum["name"] = entry.get_name(variables).replace("hyper_parameters.", "") - - if "list" in kpi.unit: - for v in value: - datum["y"] = v - data.append(datum.copy()) - else: - data.append(datum) - - return data, properties - - -class KpiPlot: - def __init__(self): - self.name = "KPI Plot" - self.id_name = self.name - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_hover(self, meta_value, variables, figure, data, click_info): - return "nothing" - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - cfg__filter_key = cfg.get("filter_key", None) - cfg__filter_value = cfg.get("filter_value", False) - cfg__kpi_name = cfg.get("kpi_name", None) - - if not cfg__kpi_name: - return None, "No KPI name passed as parameter ..." - - entries = common.Matrix.all_records(settings, setting_lists) - - has_gpu = "gpu" in ordered_vars and cfg__filter_key != "gpu" - - x_key = "gpu" if has_gpu else (ordered_vars[0] if ordered_vars else None) - - data, properties = generateData( - entries, x_key, cfg__kpi_name, variables, cfg__filter_key, cfg__filter_value - ) - - if not properties: - return None, "KPI not found ..." - - df = pd.DataFrame(data) - if df.empty: - return None, "No data available ..." - - if x_key: - df = df.sort_values(by=[x_key], ascending=True) - - y_title = properties["title"] - y_units = properties["unit"] - - is_list_kpi = "list" in properties["type"] - - y_key = "y" - if has_gpu: - do_line_plot = True - elif len(variables) == 1: - do_line_plot = all(isinstance(v, numbers.Number) for v in list(variables.values())[0]) - elif x_key is None: - do_line_plot = False - elif x_key.startswith("hyper_parameters."): - do_line_plot = True - elif is_list_kpi: - do_line_plot = True - else: - do_line_plot = False - - text = None if len(variables) > 3 else "text" - if do_line_plot: - color = None if (len(variables) == 1) else "name" - - fig = px.line(df, hover_data=df.columns, x=x_key, y=y_key, color=color, text=text) - - for i in range(len(fig.data)): - if is_list_kpi: - fig.data[i].update(mode="markers") - else: - fig.data[i].update(mode="lines+markers+text") - - fig.update_yaxes(rangemode="tozero") - - fig.update_traces(textposition="top center") - - else: - df = df.sort_values(by=["name"], ascending=True) - fig = px.bar( - df, - hover_data=df.columns, - x=x_key, - y=y_key, - color="name", - barmode="group", - text=text, - ) - - if has_gpu: - fig.update_xaxes(title="Number of GPUs used for the fine-tuning") - else: - fig.update_xaxes(title=x_key) - - x_name = x_key.replace("hyper_parameters.", "") if x_key else "single expe" - - y_lower_better = True - what = f", in {y_units}" - - y_title = f"Fine-tuning {y_title}{what}. " - title = y_title + "
    " + ("Lower is better" if y_lower_better else "Higher is better") - if is_list_kpi: - title += " (list KPI)" - fig.update_yaxes( - title=("❮ " if y_lower_better else "") + y_title + (" ❯" if not y_lower_better else "") - ) - fig.update_layout( - title=title, - title_x=0.5, - ) - fig.update_layout(legend_title_text="Configuration") - - if len(variables) == 1: - fig.layout.update(showlegend=False) - fig.update_xaxes(title=x_name) - # ❯ or ❮ - msg = [] - - return fig, msg - - -class KpiPlotReport: - def __init__(self): - self.name = "report: KPI Plot Report" - self.id_name = self.name - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - - def do_plot(self, *args): - header = [] - - header += [html.P("These plots show a summary of the Prometheus metrics")] - header += [html.H2("KPI Plot Report")] - - ordered_vars, settings, _setting_lists, variables, cfg = args - setting_lists = copy.deepcopy(_setting_lists) - - if not common.Matrix.count_records(settings, setting_lists): - header.append(html.B("No record found ...")) - return None, header - - first_entry = next(common.Matrix.all_records(settings, setting_lists)) - - for kpi_name in first_entry.results.lts.kpis.keys(): - plot_name = "KPI Plot" - header += [html.H3(f"• {kpi_name}")] - - kpi_args = report.set_config(dict(kpi_name=kpi_name), args) - for _ in range(len(ordered_vars)): - first_var = ordered_vars[0] - header += [html.H3(f"by {first_var}")] - header += report.Plot_and_Text("KPI Plot", kpi_args) - ordered_vars.append(ordered_vars.pop(0)) - - return None, header diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_table.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_table.py deleted file mode 100644 index 87c1f3463..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/kpi_table.py +++ /dev/null @@ -1,187 +0,0 @@ -import copy -import logging -from collections import defaultdict - -import matrix_benchmarking.analyze.report as analyze_report -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.table_stats as table_stats -from dash import html - - -def register(): - KPITableReport("KPI Table Report by labels", labels_version=True) - KPITableReport("KPI Table Report by settings", settings_version=True) - - -class KPITableReport: - def __init__(self, name, labels_version=False, settings_version=False): - self.name = f"report: {name}" - self.title = name - self.labels_version = labels_version - self.settings_version = settings_version - - self.id_name = self.name - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - - def do_hover(self, meta_value, variables, figure, data, click_info): - return "nothing" - - def do_plot(self, *args): - header = [html.H1(self.title)] - ordered_vars, settings, setting_lists, variables, cfg = args - header += [html.P(f"Showing {common.Matrix.count_records()} tests.")] - - # --- # - - all_kpi_settings = defaultdict(set) - for entry in common.Matrix.all_records(settings, setting_lists): - dest_all_kpi_settings = all_kpi_settings - - entry_kpi_settings = entry.results.lts.metadata.settings - - for k, v in entry_kpi_settings.__dict__.items(): - if isinstance(v, list): - v = str(v) - if not v.__hash__: - continue - - if k in common.LTS_META_KEYS: - continue - dest_all_kpi_settings[k].add(v) - - variable_labels = [] - fix_labels = [] - for k, v in all_kpi_settings.items(): - (variable_labels if len(v) > 1 else fix_labels).append(k) - - kpis_common_prefix = None - - all_kpi_results_data = [] - all_test_results_data = [] - kpis = None - idx = 0 - - unique_kpi_labels = set() - for entry in common.Matrix.all_records(settings, setting_lists): - idx += 1 - - metadata = copy.copy(entry.results.lts.metadata) - - entry_kpi_results = { - "short uuid": str(metadata.test_uuid).partition("-")[0], - "name": entry.get_name(variables) if variables else "Unique test", - } - metadata_column_count = len(entry_kpi_results) - entry_test_results = entry_kpi_results.copy() - entry_test_results.pop("name") - - metadata_settings = copy.copy(metadata.__dict__.pop("settings")) - for var in variable_labels: - entry_kpi_results[var] = metadata_settings.__dict__.get(var, "missing") - - for var in variables: - entry_test_results[var] = entry.settings.__dict__[var] - - kpis = entry.results.lts.kpis - - if kpis_common_prefix is None: - kpis_common_prefix = analyze_report.longestCommonPrefix(list(kpis.keys())) - - for kpi_name, kpi in kpis.items(): - if isinstance(kpi.value, list): - continue - - current_value_str = analyze_report.format_kpi_value(kpi) - - if (lower_better := getattr(kpi, "lower_better", None)) is not None: - direction = " ⮟" if lower_better else " ⮝" - else: - direction = "" - - kpi_value = analyze_report.OvervallResult( - 0, - "not used", - True, - current_value_str=current_value_str, - ) - - key_name = kpi_name.replace(kpis_common_prefix, "") - if len(key_name) < 5: - key_name = kpi_name - key_name += direction - entry_kpi_results[key_name] = kpi_value - entry_test_results[key_name] = kpi_value - unique_kpi_labels.add(" ".join(f"{k}={entry_kpi_results[k]}" for k in variable_labels)) - all_kpi_results_data.append(entry_kpi_results) - all_test_results_data.append(entry_test_results) - - kpi_names = ( - set( - list(all_kpi_results_data[0].keys())[ - max([metadata_column_count, len(variable_labels) + 1]) : - ] - ) - if all_kpi_results_data - else set() - ) - - if self.settings_version: - header += [html.H2("Test Settings Overview")] - - all_settings = common.Matrix.settings.copy() - all_settings.pop("stats") - header.append(analyze_report._generate_configuration_overview(all_settings, variables)) - - header.append(html.H2("KPI Results overview")) - - header.append( - analyze_report._generate_results_overview( - all_test_results_data, ordered_vars, kpi_names, kpis_common_prefix, warn=False - ) - ) - - if self.labels_version: - header += [html.H2("KPI Labels Overview")] - header.append( - analyze_report._generate_configuration_overview(all_kpi_settings, variable_labels) - ) - - header.append(html.H2("KPI Results overview")) - if len(unique_kpi_labels) != len(all_test_results_data): - msg = f"found {len(unique_kpi_labels)} unique KPI labels for {len(all_test_results_data)} test entries." - logging.warning(msg) - header.append(html.Span([html.B("WARNING:"), msg])) - - header.append( - analyze_report._generate_results_overview( - all_kpi_results_data, variable_labels, kpi_names, kpis_common_prefix - ) - ) - - header.append(html.H2("KPIs description")) - - for kpi_name, kpi in kpis.items(): - header.append(html.B(kpi_name)) - details_elts = [ - html.Li(html.I(kpi.help)), - html.Li(f"in {kpi.unit}"), - ] - - if divisor_unit := getattr(kpi, "divisor_unit", None): - details_elts += [ - html.Li(f"converted into {divisor_unit}"), - ] - if (lower_better := getattr(kpi, "lower_better", None)) is not None: - details_elts += [ - html.Li("lower is better (⮟)" if lower_better else "higher is better (⮝)"), - ] - header.append(html.Ul(details_elts)) - header.append(html.Br()) - - if not kpis: - header.append(html.P("No KPIs found ...")) - - return None, header diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/lts_documentation.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/lts_documentation.py deleted file mode 100644 index de80727ea..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/lts_documentation.py +++ /dev/null @@ -1,116 +0,0 @@ -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.table_stats as table_stats -import pandas as pd -from dash import html - - -def register(): - LtsDocumentationReport() - - -class LtsDocumentationReport: - def __init__(self): - self.name = "report: LTS Documentation" - self.id_name = self.name - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - - def do_hover(self, meta_value, variables, figure, data, click_info): - return "nothing" - - def do_plot(self, *args): - header = [] - ordered_vars, settings, setting_lists, variables, cfg = args - - kpi_data = [] - for entry in common.Matrix.all_records(settings, setting_lists): - header += [ - html.H1(entry.get_name(variables or [s for s in settings.keys() if s != "stats"])) - ] - header += generateOneLtsDocumentationReport(entry, kpi_data) - header += [html.Hr()] - - kpi_df = pd.DataFrame(kpi_data) - - if not kpi_df.empty: - header += [html.H1("CSV summary")] - # https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html - header += [html.Code(kpi_df.to_csv(index=False), style={"white-space": "pre-wrap"})] - - return None, header - - -def generateOneLtsDocumentationReport(entry, kpi_data): - lts = entry.results.lts - - header = [] - header += [html.H2("metadata")] - metadata = [] - - metadata += [ - html.Li( - [ - html.B("urls:"), - html.Ul( - [html.Li(html.A(k, href=v)) for k, v in lts.metadata.settings.urls.items()] - ), - ] - ) - ] - metadata += [html.Li([html.B("start:"), html.Code(lts.metadata.start)])] - metadata += [html.Li([html.B("presets:"), html.Code(", ".join(lts.metadata.presets))])] - metadata += [ - html.Li( - [ - html.B("labels:"), - html.Ul( - [ - html.Li(f"{k}: {v}") - for k, v in lts.metadata.settings.__dict__.items() - if k not in common.LTS_META_KEYS - ] - ), - ] - ) - ] - - header += [html.Ul(metadata)] - - header += [html.H2("kpis")] - kpis = [] - - kpi_entry = dict() - kpi_data.append(kpi_entry) - for name, kpi in lts.kpis.items(): - kpis += [ - html.Li( - [ - html.P( - [ - html.B(name), - "⮕", - html.Code(f"{kpi.value} {kpi.unit}"), - html.Br(), - html.I(kpi.help), - html.Br(), - ] - ) - ] - ) - ] - - kpi_entry |= kpi.__dict__ - - timestamp = kpi_entry.pop("@timestamp") - kpi_entry["timestamp"] = str(timestamp) - - for meta_key in common.LTS_META_KEYS: - kpi_entry.pop(meta_key, None) - - kpi_entry[name] = kpi.value - - header += [html.Ul(kpis)] - - return header diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/__init__.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/__init__.py deleted file mode 100644 index 780c76e0d..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/__init__.py +++ /dev/null @@ -1,258 +0,0 @@ -import datetime -from collections import defaultdict - -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.table_stats as table_stats -import pandas as pd -import plotly.express as px -import plotly.graph_objs as go -from dash import html - - -def default_get_metrics(entry, metric): - return entry.results.metrics[metric] - - -def get_tests_timestamp_plots(entries, y_max, variables): - data = [] - tests_timestamp_y_position = [-y_max * 0.25] * 3 - - now = datetime.datetime.now() - tz_offset = datetime.datetime.fromtimestamp( - now.timestamp() - ) - datetime.datetime.utcfromtimestamp(now.timestamp()) - - for entry in entries: - if "tests_timestamp" not in entry.results.__dict__: - continue - - for test_timestamp in entry.results.tests_timestamp: - # convert to local timezone and remove timezone info - start = test_timestamp.start + tz_offset - start.replace(tzinfo=None) - end = test_timestamp.end + tz_offset - end.replace(tzinfo=None) - - settings_text_lst = [ - str(v) if k.startswith("*") else f"{k}={v}" - for k, v in test_timestamp.settings.items() - if k in variables - ] - if not settings_text_lst: - settings_text_lst = [ - str(v) if k.startswith("*") else f"{k}={v}" - for k, v in test_timestamp.settings.items() - ] - - mid = start + (end - start) / 2 - data.append( - go.Scatter( - x=[start, mid, end], - y=tests_timestamp_y_position, - name="Test timestamps", - hoverlabel={"namelength": -1}, - hovertext="
    ".join(f"{k}={v}" for k, v in test_timestamp.settings.items()), - legendgroup="Test timestamps", - legendgrouptitle_text="Test timestamps", - text=["", "
    ".join(settings_text_lst) + "
    ", ""], - mode="lines+text", - textposition="top center", - showlegend=False, - line=dict(width=15), - ) - ) - - return tests_timestamp_y_position[0], data - - -class Plot: - def __init__( - self, - metrics, - name, - title, - y_title, - get_metrics=default_get_metrics, - filter_metrics=lambda entry, metrics: metrics, - as_timestamp=False, - get_legend_name=None, - show_metrics_in_title=False, - show_queries_in_title=False, - show_legend=True, - y_divisor=1, - higher_better=False, - substract_first=False, - ): - self.name = name - self.id_name = f"prom_overview_{''.join(c for c in self.name if c not in '?:!/;()').replace(' ', '_').lower()}" - self.title = title - self.metrics = metrics - self.y_title = y_title - self.y_divisor = y_divisor - self.filter_metrics = filter_metrics - self.get_metrics = get_metrics - self.as_timestamp = as_timestamp - self.get_legend_name = get_legend_name - self.show_metrics_in_title = show_metrics_in_title - self.show_queries_in_title = show_queries_in_title - self.substract_first = substract_first - self.show_legend = show_legend - self.higher_better = higher_better - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_hover(self, meta_value, variables, figure, data, click_info): - return "nothing" - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - plot_title = self.title if self.title else self.name - - cfg__as_timeline = cfg.get("as_timeline", False) - - if self.show_metrics_in_title: - metric_names = [ - list(metric.items())[0][0] if isinstance(metric, dict) else metric - for metric in self.metrics.keys() - ] - plot_title += f"
    {'
    '.join(metric_names)}" - - if self.show_queries_in_title: - queries_names = self.metrics.values() - plot_title += f"
    {'
    '.join(queries_names)}" - - y_max = 0 - single_expe = common.Matrix.count_records(settings, setting_lists) == 1 - as_timeline = single_expe or cfg__as_timeline - - show_test_timestamps = True - if not as_timeline: - show_test_timestamps = False - - metric_queries = set() - metric_labels_by_group = defaultdict(set) - - data = [] - for entry in common.Matrix.all_records(settings, setting_lists): - entry_name = entry.get_name(variables) - - sort_index = ( - entry.get_settings()[ordered_vars[0]] if len(variables) == 1 else entry_name - ) - - for _metric in self.metrics: - metric_name, metric_query = ( - list(_metric.items())[0] if isinstance(_metric, dict) else (_metric, _metric) - ) - - metric_queries.add(metric_query) - for metric in self.filter_metrics(entry, self.get_metrics(entry, metric_name)): - if not metric: - continue - - x_values = [x for x, y in metric.values.items()] - y_values = [y / self.y_divisor for x, y in metric.values.items()] - - if self.substract_first and y_values: - first = y_values[0] - y_values = [y - first for y in y_values] - - if self.get_legend_name: - legend_name, legend_group = self.get_legend_name(metric_name, metric.metric) - else: - legend_name = metric.metric.get("__name__", metric_name) - legend_group = None - - if as_timeline: - if legend_group is None: - legend_group = "" - else: - legend_group += "
    " - legend_group += entry.get_name(variables) - - if legend_group: - legend_group = str(legend_group) - else: - legend_group = None - - if metric.metric: - metric_labels_by_group[legend_group].add(str(metric.metric)) - - if self.as_timestamp: - x_values = [datetime.datetime.fromtimestamp(x) for x in x_values] - else: - x_start = x_values[0] - x_values = [x - x_start for x in x_values] - - y_max = max([y_max] + y_values) - if as_timeline: - data.append( - go.Scatter( - x=x_values, - y=y_values, - name=str(legend_name), - hoverlabel={"namelength": -1}, - showlegend=self.show_legend, - legendgroup=legend_group, - legendgrouptitle_text=legend_group, - mode="markers+lines", - ) - ) - - else: - entry_version = entry.get_name(variables) - for y_value in y_values: - data.append( - dict( - Version=entry_version, - SortIndex=sort_index, - Metric=legend_name, - Value=y_value, - ) - ) - - if not data: - return None, f"No data to plot for {self.metrics} ..." - - if show_test_timestamps: - tests_timestamp_y_position, plots = get_tests_timestamp_plots( - common.Matrix.all_records(settings, setting_lists), y_max, variables - ) - data += plots - - if as_timeline: - fig = go.Figure(data=data) - - fig.update_layout( - title=plot_title, - title_x=0.5, - yaxis=dict( - title=self.y_title, - range=[tests_timestamp_y_position if show_test_timestamps else 0, y_max * 1.05], - ), - xaxis=dict(title=None if self.as_timestamp else "Time (in s)"), - ) - else: - df = pd.DataFrame(data).sort_values(by=["SortIndex"]) - fig = px.box(df, x="Version", y="Value", color="Version", points="all") - fig.update_layout(title=plot_title, title_x=0.5, yaxis=dict(title=self.y_title)) - - msg = [] - # if metric_queries: - # msg += [html.H3("Metric queries")] - - # for metric_query in metric_queries: - # msg += [html.Ul(html.Li(html.Code(str(metric_query))))] - - if metric_labels_by_group: - msg += [html.H3("Metric labels")] - - for group, labelss in metric_labels_by_group.items(): - msg += [html.H4(group if group else "no group")] - for labels in labelss: - msg += [html.Ul(html.Li(html.Code(str(labels))))] - - if msg: - msg = html.Details([html.Summary("Prometheus queries labels")] + msg) - - return fig, msg diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/cpu_memory.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/cpu_memory.py deleted file mode 100644 index 36d751bf7..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/prom/cpu_memory.py +++ /dev/null @@ -1,246 +0,0 @@ -import datetime -from collections import defaultdict - -import matrix_benchmarking.common as common -import matrix_benchmarking.plotting.table_stats as table_stats -import pandas as pd -import plotly.express as px -import plotly.graph_objs as go -from dash import html - -from . import get_tests_timestamp_plots - - -def default_get_metrics(entry, metric): - return entry.results.metrics[metric] - - -class Plot: - def __init__( - self, - metrics, - y_title, - get_metrics=default_get_metrics, - filter_metrics=lambda entry, metrics: metrics, - as_timestamp=False, - container_name="all", - is_memory=False, - is_cluster=False, - skip_nodes=None, - ): - self.name = f"Prom: {y_title}" - - self.id_name = f"prom_overview_{y_title}" - self.metrics = metrics - self.y_title = y_title - self.filter_metrics = filter_metrics - self.get_metrics = get_metrics - self.as_timestamp = as_timestamp - self.container_name = container_name - self.is_memory = is_memory - self.is_cluster = is_cluster - - if skip_nodes is not None: - self.skip_nodes = skip_nodes - else: - self.skip_nodes = self.is_memory and not self.is_cluster - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_hover(self, meta_value, variables, figure, data, click_info): - return "nothing" - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - cfg__as_timeline = cfg.get("as_timeline", False) - - fig = go.Figure() - metric_names = [ - list(metric.items())[0][0] if isinstance(metric, dict) else metric - for metric in self.metrics - ] - - plot_title = f"Prometheus: {self.y_title}" - y_max = 0 - - y_divisor = 1024 * 1024 * 1024 if self.is_memory else 1 - - single_expe = common.Matrix.count_records(settings, setting_lists) == 1 - - as_timeline = single_expe or cfg__as_timeline - - show_test_timestamps = True - if not as_timeline: - show_test_timestamps = False - - data = [] - data_rq = [] - data_lm = [] - - metrics_by_group = defaultdict(set) - - for entry in common.Matrix.all_records(settings, setting_lists): - entry_name = entry.get_name(variables) - - sort_index = ( - entry.get_settings()[ordered_vars[0]] if len(variables) == 1 else entry_name - ) - - for _metric in self.metrics: - metric_name, metric_query = ( - list(_metric.items())[0] if isinstance(_metric, dict) else (_metric, _metric) - ) - - for metric in self.filter_metrics(entry, self.get_metrics(entry, metric_name)): - if not metric: - continue - - x_values = [x for x, y in metric.values.items()] - y_values = [y / y_divisor for x, y in metric.values.items()] - - metric_actual_name = metric.metric.get("__name__", metric_name) - legend_name = metric_actual_name - if metric.metric.get("container") == "POD": - continue - - if "_sum_" in metric_name: - legend_group = None - legend_name = "sum(all)" - else: - legend_group = ( - metric.metric.get("pod", "") - + "/" - + metric.metric.get("container", self.container_name) - if not self.is_cluster - else None - ) - - if metric.metric: - metrics_by_group[legend_group].add(str(metric.metric)) - - if self.as_timestamp: - x_values = [datetime.datetime.fromtimestamp(x) for x in x_values] - else: - x_start = x_values[0] - x_values = [x - x_start for x in x_values] - - y_max = max([y_max] + y_values) - - opts = {} - - if self.skip_nodes and "node" not in metric.metric: - continue - - is_req_or_lim = "limit" in legend_name or "requests" in legend_name - - if as_timeline: - entry_name = "Test" - - if "requests" in metric_actual_name: - opts["line_color"] = "orange" - opts["line_dash"] = "dot" - opts["mode"] = "lines" - - elif "limits" in metric_actual_name or "capacity" in metric_actual_name: - opts["line_color"] = "red" - opts["mode"] = "lines" - opts["line_dash"] = "dash" - else: - opts["mode"] = "markers+lines" - - data.append( - go.Scatter( - x=x_values, - y=y_values, - name=legend_name, - hoverlabel={"namelength": -1}, - showlegend=True, - legendgroup=legend_group, - legendgrouptitle_text=legend_group, - **opts, - ) - ) - - else: - if is_req_or_lim: - lst = data_lm if "limits" in legend_name else data_rq - - lst.append( - dict( - Version=entry_name, - SortIndex=sort_index, - Value=y_values[0], - Metric=legend_name, - ) - ) - - else: - for y_value in y_values: - data.append( - dict( - Version=entry_name, - SortIndex=sort_index, - Metric=legend_name, - Value=y_value, - ) - ) - - if not data: - return None, f"No data to plot for {self.metrics} ..." - - if show_test_timestamps: - tests_timestamp_y_position, plots = get_tests_timestamp_plots( - common.Matrix.all_records(settings, setting_lists), y_max, variables - ) - data += plots - - if as_timeline: - fig = go.Figure(data=data) - - fig.update_layout( - title=plot_title, - title_x=0.5, - yaxis=dict( - title=self.y_title + (" (in Gi)" if self.is_memory else ""), - range=[tests_timestamp_y_position if show_test_timestamps else 0, y_max * 1.05], - ), - xaxis=dict(title="Time (in s)"), - ) - else: - df = pd.DataFrame(data).sort_values(by=["SortIndex"]) - - fig = px.box(df, x="Version", y="Value", color="Version") - fig.update_layout( - title=plot_title, - title_x=0.5, - yaxis=dict(title=self.y_title + (" (in Gi)" if self.is_memory else "")), - ) - if data_rq: - df_rq = pd.DataFrame(data_rq).sort_values(by=["SortIndex"]) - fig.add_scatter( - name="Request", - x=df_rq["Version"], - y=df_rq["Value"], - mode="lines", - line=dict(color="orange", width=5, dash="dot"), - ) - if data_lm: - df_lm = pd.DataFrame(data_lm).sort_values(by=["SortIndex"]) - fig.add_scatter( - name="Limit", - x=df_lm["Version"], - y=df_lm["Value"], - mode="lines", - line=dict(color="red", width=5, dash="dot"), - ) - - msg = [] - for group, queries in metrics_by_group.items(): - msg += [html.H4(group if group else "no group")] - for query in queries: - msg += [html.Ul(html.Li(html.Code(str(query))))] - - if msg: - msg = html.Details([html.Summary("Prometheus queries labels")] + msg) - - return fig, msg diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/report.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/report.py deleted file mode 100644 index 56f38bb0d..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/report.py +++ /dev/null @@ -1,95 +0,0 @@ -import copy -import logging - -import matrix_benchmarking.plotting.table_stats as table_stats -from dash import dcc, html - - -def set_config(additional_cfg, args): - cfg = copy.deepcopy(args[-1]) - cfg.d.update(additional_cfg) - return list(args[:-1]) + [cfg] - - -def set_settings(new_settings, args): - """ - Set specific settings and remove them from variables to fix the plot dimensions - - Args: - new_settings: Dict of settings to apply (e.g., {"model": "gpt-oss-120b"}) - args: Original plot arguments - - Returns: - Updated args with settings applied and variables modified - """ - # Make deep copies to avoid modifying the original - args_copy = copy.deepcopy(args) - ordered_vars, settings, setting_lists, variables, cfg = args_copy - - # Update settings with new values - settings.update(new_settings) - - # Remove the keys from variables so they're no longer treated as varying dimensions - for key in new_settings.keys(): - if key in variables: - del variables[key] - - # Update setting_lists to remove dimensions that are now fixed - # setting_lists is a list of lists like: - # [[('model', 'gpt-oss-120b'), ('model', 'llama3.3-70b')], [('load_shape', 'Heterogeneous'), ('load_shape', 'Multiturn')]] - updated_setting_lists = [] - for setting_list in setting_lists: - # Check if this setting_list corresponds to a dimension we're fixing - if setting_list and setting_list[0][0] not in new_settings: - # Keep this dimension since we're not fixing it - updated_setting_lists.append(setting_list) - - # Replace the setting_lists with the filtered version - setting_lists[:] = updated_setting_lists - - return ordered_vars, settings, setting_lists, variables, cfg - - -def set_entry(entry, _args): - args = copy.deepcopy(_args) - ordered_vars, settings, setting_lists, variables, cfg = args - - settings.update(entry.settings.__dict__) - setting_lists[:] = [] - variables.clear() - return args - - -def Plot(name, args, msg_p=None): - try: - stats = table_stats.TableStats.stats_by_name[name] - except KeyError: - logging.error(f"Report: Stats '{name}' does not exist. Skipping it.") - stats = None - - fig, msg = stats.do_plot(*args) if stats else (None, f"Stats '{name}' does not exit :/") - - if msg_p is not None: - msg_p.append(msg) - - return dcc.Graph(figure=fig) - - -def Plot_and_Text(name, args): - msg_p = [] - - data = [Plot(name, args, msg_p)] - - if msg_p[0]: - data.append( - html.Div( - msg_p[0], - style={ - "border-radius": "5px", - "padding": "0.5em", - "background-color": "lightgray", - }, - ) - ) - - return data diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/styles_css.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/styles_css.py deleted file mode 100644 index 90924cc21..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/styles_css.py +++ /dev/null @@ -1,414 +0,0 @@ -EMBEDDED_CSS = """ -/* Base responsive container */ -.report-container { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; - background-color: #f8f9fa; - color: #343a40; - padding: 2rem; - width: 95%; - max-width: 1400px; - overflow-x: visible; - margin: 0 auto; -} - -/* Responsive styles for plots grid */ -@media (max-width: 1024px) { - .plots-grid-responsive { - grid-template-columns: 1fr !important; - } -} - -/* Additional responsive styles for mobile */ -@media (max-width: 768px) { - .report-container { - padding: 1rem !important; - width: 95% !important; - } -} - -@media (max-width: 480px) { - .info-card-header { - font-size: 1rem !important; - padding: 0.75rem 1rem !important; - } - - .info-item { - padding: 0.75rem 1rem !important; - } - - .info-key { - width: 100px !important; - font-size: 0.85rem !important; - padding-right: 0.5rem !important; - } - - .info-value { - font-size: 0.8rem !important; - } - - .plot-content { - padding: 1rem !important; - } - - .plot-container { - padding: 1rem !important; - } - - .plot-container img { - min-width: 300px !important; - max-width: none !important; - width: auto !important; - } -} - -/* Plot specific styling */ -.plot-container { - width: 100% !important; - height: auto !important; - display: flex !important; - justify-content: center !important; - align-items: center !important; - padding: 2rem !important; - overflow: visible !important; -} - -.plot-container > div { - width: 100% !important; - height: auto !important; -} - -.plot-container p { - margin: 0 !important; - width: 100% !important; -} - -.plot-container a { - display: block !important; - width: 100% !important; -} - -.plot-container img { - max-width: none !important; - width: auto !important; - height: auto !important; - display: block !important; - object-fit: contain !important; - margin: 0 auto !important; - min-width: 600px !important; -} -""" - -STYLE_CONTAINER = { - "font-family": '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif', - "background-color": "#f8f9fa", - "color": "#343a40", - "padding": "2rem", - "width": "95%", - "max-width": "1400px", - "overflow-x": "visible", - "margin": "0 auto", -} - -STYLE_H1 = { - "font-size": "2.5rem", - "color": "#007BFF", - "text-align": "center", - "margin-bottom": "2.5rem", - "font-weight": "300", -} - -STYLE_H2_SECTION = { - "font-size": "1.75rem", - "color": "#343a40", - "border-bottom": "2px solid #dee2e6", - "padding-bottom": "0.5rem", - "margin": "2.5rem 0 1.5rem 0", -} - -STYLE_INFO_CARD = { - "background": "white", - "border": "1px solid #dee2e6", - "border-radius": "8px", - "box-shadow": "0 2px 8px rgba(0, 0, 0, 0.08)", - "overflow": "hidden", - "display": "flex", - "flex-direction": "column", - "margin-bottom": "1rem", -} - -STYLE_INFO_ROW = { - "display": "grid", - "grid-template-columns": "repeat(auto-fit, minmax(300px, 1fr))", - "gap": "1.5rem", - "margin-bottom": "2rem", -} - -STYLE_INFO_CARD_HEADER = { - "background": "#f8f9fa", - "border-bottom": "1px solid #dee2e6", - "padding": "1rem 1.25rem", - "font-size": "1.1rem", - "font-weight": "600", - "color": "#007BFF", -} - -STYLE_INFO_CARD_BODY = {"flex": "1", "padding": "0"} - -STYLE_INFO_ITEM = { - "display": "flex", - "align-items": "flex-start", - "padding": "0.75rem 1.25rem", - "border-bottom": "1px solid #f8f9fa", - "min-height": "2.5rem", -} - -STYLE_INFO_ITEM_LAST = { - "display": "flex", - "align-items": "flex-start", - "padding": "0.75rem 1.25rem", - "border-bottom": "none", - "min-height": "2.5rem", -} - -STYLE_INFO_KEY = { - "font-weight": "600", - "color": "#495057", - "width": "120px", - "flex-shrink": "0", - "font-size": "0.9rem", - "text-align": "left", - "padding-right": "1rem", -} - -STYLE_INFO_VALUE = { - "font-family": '"SF Mono", Monaco, "Cascadia Code", "Roboto Mono", Consolas, "Courier New", monospace', - "font-size": "0.85rem", - "color": "#343a40", - "text-align": "left", - "flex": "1", - "overflow-wrap": "break-word", - "word-break": "break-word", - "line-height": "1.4", -} - -STYLE_INFO_VALUE_HIGHLIGHT = { - "font-family": '"SF Mono", Monaco, "Cascadia Code", "Roboto Mono", Consolas, "Courier New", monospace', - "font-size": "1rem", - "color": "#28a745", - "text-align": "left", - "flex": "1", - "overflow-wrap": "break-word", - "word-break": "break-word", - "line-height": "1.4", - "font-weight": "600", -} - -STYLE_PLOTS_GRID = { - "display": "grid", - "grid-template-columns": "1fr", - "gap": "3rem", - "margin-bottom": "3rem", - "padding": "1rem 0", -} - -STYLE_PLOT_CARD = { - "background": "white", - "border": "1px solid #dee2e6", - "border-radius": "8px", - "box-shadow": "0 2px 8px rgba(0, 0, 0, 0.08)", - "overflow": "visible", - "width": "100%", - "display": "flex", - "flex-direction": "column", - "margin-bottom": "2rem", - "min-width": "0", -} - -STYLE_PLOT_CONTENT = { - "padding": "2rem", - "width": "100%", - "display": "flex", - "flex-direction": "column", - "align-items": "center", - "justify-content": "flex-start", - "overflow": "visible", -} - -STYLE_DETAILS = { - "background": "white", - "border": "1px solid #dee2e6", - "border-radius": "8px", - "box-shadow": "0 2px 8px rgba(0, 0, 0, 0.08)", - "overflow": "hidden", - "margin-bottom": "2rem", -} - -STYLE_DETAILS_SUMMARY = { - "background": "#f8f9fa", - "padding": "1rem 1.25rem", - "cursor": "pointer", - "font-weight": "600", - "color": "#007BFF", - "outline": "none", - "border": "none", -} - -STYLE_DETAILS_CONTENT = {"padding": "1.25rem"} - -STYLE_BENCHMARK_SECTION = {"margin-bottom": "2rem"} - -STYLE_JSON_PRE = { - "background": "#2d3748", - "color": "#e2e8f0", - "padding": "1rem", - "border-radius": "6px", - "overflow-x": "auto", - "font-size": "0.8rem", - "line-height": "1.5", - "white-space": "pre-wrap", - "word-break": "break-all", -} - -STYLE_SMALL_TEXT = {"color": "#6c757d", "font-size": "0.8rem"} - -STYLE_H4 = {"margin-top": "1rem"} - -# Comparison report styles -COMPARISON_CSS = """ -/* Comparison table styles */ -.comparison-table { - width: 100%; - border-collapse: collapse; - margin-bottom: 1rem; - background: white; - border-radius: 8px; - overflow: hidden; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); -} - -.comparison-table th, -.comparison-table td { - padding: 0.75rem 1rem; - text-align: left; - border-bottom: 1px solid #dee2e6; - vertical-align: top; -} - -.comparison-table th { - background: #f8f9fa; - font-weight: 600; - color: #343a40; - position: sticky; - top: 0; - z-index: 10; -} - -.comparison-table td { - background: white; -} - -.comparison-table tr:hover { - background: #f8f9fa; -} - -.comparison-table-highlight { - background: #e3f2fd !important; - font-weight: 600; -} - -/* Responsive table styling */ -@media (max-width: 768px) { - .comparison-table { - font-size: 0.85rem; - } - - .comparison-table th, - .comparison-table td { - padding: 0.5rem; - } -} - -/* Comparison section styling */ -.comparison-section { - background: white; - border-radius: 8px; - padding: 1.5rem; - margin-bottom: 2rem; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); -} - -/* Error message styling */ -.error-message { - background: #f8d7da; - color: #721c24; - padding: 1rem; - border-radius: 8px; - border: 1px solid #f5c6cb; - text-align: center; - margin: 1rem 0; -} -""" - -STYLE_COMPARISON_TABLE = { - "width": "100%", - "border-collapse": "collapse", - "margin-bottom": "1rem", - "background": "white", - "border-radius": "8px", - "overflow": "hidden", - "box-shadow": "0 2px 8px rgba(0, 0, 0, 0.08)", -} - -STYLE_TABLE_HEADER = { - "padding": "0.75rem 1rem", - "text-align": "left", - "border-bottom": "1px solid #dee2e6", - "background": "#f8f9fa", - "font-weight": "600", - "color": "#343a40", - "position": "sticky", - "top": "0", - "z-index": "10", -} - -STYLE_TABLE_CELL = { - "padding": "0.75rem 1rem", - "text-align": "left", - "border-bottom": "1px solid #dee2e6", - "background": "white", - "vertical-align": "top", -} - -STYLE_TABLE_CELL_HIGHLIGHT = { - "padding": "0.75rem 1rem", - "text-align": "left", - "border-bottom": "1px solid #dee2e6", - "background": "#e3f2fd", - "font-weight": "600", - "vertical-align": "top", -} - -STYLE_COMPARISON_SECTION = { - "background": "white", - "border-radius": "8px", - "padding": "1.5rem", - "margin-bottom": "2rem", - "box-shadow": "0 2px 8px rgba(0, 0, 0, 0.08)", -} - -STYLE_H3 = { - "font-size": "1.5rem", - "color": "#343a40", - "margin": "1.5rem 0 1rem 0", - "font-weight": "500", -} - -STYLE_ERROR_MESSAGE = { - "background": "#f8d7da", - "color": "#721c24", - "padding": "1rem", - "border-radius": "8px", - "border": "1px solid #f5c6cb", - "text-align": "center", - "margin": "1rem 0", -} diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py deleted file mode 100644 index cd89d36bc..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py +++ /dev/null @@ -1,46 +0,0 @@ -from math import log - - -def human_readable_size(value): - """Convert a byte value to a human-readable string. - - Args: - value (int): The byte value to convert. - - Returns: - str: The human-readable string representation of the byte value. - """ - if value is None: - return "N/A" - - suffix = [" kB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB", " RB", " QB"] - base = 1000 - bytes_ = float(value) - abs_bytes = abs(bytes_) - if abs_bytes == 1: - return f"{int(bytes_)} Byte" - elif abs_bytes < base: - return f"{int(bytes_)} Bytes" - exp = int(min(log(abs_bytes, base), len(suffix))) - return f"{(bytes_ / (base**exp)):.1f}{suffix[exp - 1]}" - - -def format_duration(seconds): - """Convert a duration in seconds to a human-readable string. - - Args: - seconds (float): The duration in seconds. - - Returns: - str: The human-readable string representation of the duration. - """ - if seconds is None: - return "N/A" - if seconds < 1e-6: - return f"{seconds * 1e9:.2f} ns" - if seconds < 1e-3: - return f"{seconds * 1e6:.2f} µs" - if seconds < 1: - return f"{seconds * 1e3:.2f} ms" - - return f"{seconds:.2f} s" diff --git a/projects/matrix_benchmarking/visualizations/helpers/store/__init__.py b/projects/matrix_benchmarking/visualizations/helpers/store/__init__.py deleted file mode 100644 index 6c6bf1a45..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/store/__init__.py +++ /dev/null @@ -1,319 +0,0 @@ -import fnmatch -import functools -import inspect -import json -import logging -import os -import pathlib -import pickle -import types - -import jsonpath_ng -import matrix_benchmarking.common as common -import matrix_benchmarking.store as store -from matrix_benchmarking.parse import json_dumper - - -class BaseStore: - def __init__( - self, - *, - cache_filename, - important_files, - artifact_dirnames, - artifact_paths, - extra_mandatory_files=[], - parse_always, - parse_once, - generate_lts_payload=None, - lts_payload_model=None, - models_kpis=None, - get_kpi_labels=None, - ): - self.cache_filename = cache_filename - self.important_files = important_files - self.extra_mandatory_files = extra_mandatory_files - - self.artifact_dirnames = artifact_dirnames - self.artifact_paths = artifact_paths - - self.parse_always = parse_always - self.parse_once = parse_once - - self.lts_payload_model = lts_payload_model - if lts_payload_model: - self.generate_lts_payload = generate_lts_payload - store.register_lts_schema(lts_payload_model) - self.models_kpis = models_kpis - self.get_kpi_labels = get_kpi_labels - - def is_mandatory_file(self, filename): - return ( - filename.name in ("settings", "exit_code", "config.yaml", "skip") - or filename.name in self.extra_mandatory_files - or filename.name.startswith("settings.") - ) - - def is_important_file(self, filename): - if str(filename) in self.important_files: - return True - - for important_file in self.important_files: - if "*" not in important_file: - continue - - if fnmatch.filter([str(filename)], important_file): - return True - - return False - - def is_cache_file(self, filename): - return filename.name == self.cache_filename - - def register_important_file(self, base_dirname, filename): - to_return = base_dirname / filename - if self.is_important_file(filename): - return to_return - - filename_resolved = (base_dirname / filename).resolve().relative_to(base_dirname.resolve()) - if self.is_important_file(filename_resolved): - # filename is composed of .. - # accept it - return to_return - - logging.warning(f"File '{filename}' not part of the important file list :/") - if pathlib.Path(filename).is_absolute(): - logging.warning( - f"File '{filename}' is an absolute path. Should be relative to {base_dirname}." - ) - - return to_return - - def resolve_artifact_dirnames(self, dirname): - artifact_paths = types.SimpleNamespace() - for artifact_dirname, unresolved_dirname in self.artifact_dirnames.__dict__.items(): - direct_resolution = dirname / unresolved_dirname - if direct_resolution.exists(): - artifact_paths.__dict__[artifact_dirname] = pathlib.Path(unresolved_dirname) - continue - - resolutions = list(dirname.glob(unresolved_dirname)) - resolved_dir = None - - if direct_resolution.exists(): - # all good - resolved_dir = direct_resolution - elif not resolutions: - logging.warning( - f"Cannot resolve {artifact_dirname} glob '{unresolved_dirname}' in '{dirname}'" - ) - else: - if len(resolutions) > 1: - logging.debug( - f"Found multiple resolutions for {artifact_dirname} glob '{unresolved_dirname}' in '{dirname}': {resolutions}. Taking the last one" - ) - resolved_dir = [r.relative_to(dirname) for r in sorted(resolutions)][-1] - else: - resolved_dir = resolutions[0].relative_to(dirname) - - artifact_paths.__dict__[artifact_dirname] = resolved_dir - - self.artifact_paths.__dict__.update(artifact_paths.__dict__) - - def load_cache(self, dirname): - try: - with open(dirname / self.cache_filename, "rb") as f: - results = pickle.load(f) - except FileNotFoundError: - raise # will be catch at higher levels - except Exception as e: - logging.error("Could not reload the cache file: {e}") - raise e - - self._prepare_after_pickle(results) - self.prepare_after_pickle(results) - - return results - - def parse_directory(self, fn_add_to_matrix, dirname, import_settings, exit_code): - ignore_cache = os.environ.get("MATBENCH_STORE_IGNORE_CACHE", False) in ( - "yes", - "y", - "true", - "True", - ) - if not ignore_cache: - try: - results = self.load_cache(dirname) - except FileNotFoundError: - results = None # Cache file doesn't exit, ignore and parse the artifacts - else: - logging.info("MATBENCH_STORE_IGNORE_CACHE is set, not processing the cache file.") - results = None - - if results: - # reloaded from cache - self.parse_always(results, dirname, import_settings) - self.parse_lts(results, import_settings, exit_code) - - fn_add_to_matrix(results) - - return - - self.resolve_artifact_dirnames(dirname) - - results = types.SimpleNamespace() - - parse_once_kwargs = {} - if "import_settings" in inspect.signature(self.parse_once).parameters: - parse_once_kwargs["import_settings"] = import_settings - - self.parse_once(results, dirname, **parse_once_kwargs) - self.parse_always(results, dirname, import_settings) - self.parse_lts(results, import_settings, exit_code) - - fn_add_to_matrix(results) - - with open(dirname / self.cache_filename, "wb") as f: - self.prepare_for_pickle(results) - self._prepare_for_pickle(results) - - pickle.dump(results, f) - - self._prepare_after_pickle(results) - self.prepare_after_pickle(results) - - print("parsing done :)") - - def parse_lts(self, results, import_settings, exit_code): - if not self.lts_payload_model: - return - - results.lts = lts_payload = self.generate_lts_payload(results, import_settings) - if self.models_kpis: - lts_payload.kpis = self.generate_lts_kpis(lts_payload) - - lts_payload.metadata.exit_code = exit_code - - # this allows the LTS payload to tell that it shouldn't be validated - # to be improved and streamlined - if hasattr(lts_payload, "is_valid"): - return - - validate_lts_payload(self.lts_payload_model, lts_payload, import_settings, reraise=False) - - def generate_lts_kpis(self, lts_payload): - kpis = {} - for name, properties in self.models_kpis.items(): - kpi = {} | properties | self.get_kpi_labels(lts_payload) - - kpi_func = kpi.pop("__func__") - try: - kpi["value"] = kpi_func(lts_payload) - except Exception as e: - if isinstance(e, KeyboardInterrupt): - raise - - msg = f"Failed to generate KPI {name}: {e}" - # don't write an error (TOPSAIL detects it and fails - # the test) if the LTS has already been marked as - # invalid - is_valid = getattr(lts_payload, "is_valid", True) - (logging.error if is_valid else logging.warning)(msg) - - kpi["value"] = None - - kpis[name] = types.SimpleNamespace(**kpi) - - return kpis - - def _prepare_for_pickle(self, results): - pass - - def _prepare_after_pickle(self, results): - pass - - def prepare_for_pickle(self, results): - pass - - def prepare_after_pickle(self, results): - pass - - def parse_data(): - # delegate the parsing to the simple_store - store.register_custom_rewrite_settings(_rewrite_settings) - store_simple.register_custom_parse_results(_parse_directory) - - from . import lts - - store_simple.register_custom_build_lts_payloads(lts.build_lts_payloads) - - return store_simple.parse_data() - - def build_lts_payloads(self): - if not self.lts_payload_model: - raise ValueError( - "Cannot build the LTS payload: no `lts_payload_model` defined in this workload." - ) - - for entry in common.Matrix.processed_map.values(): - results = entry.results - lts_payload = results.lts - - validate_lts_payload( - self.lts_payload_model, lts_payload, entry.import_settings, reraise=True - ) - - yield lts_payload, lts_payload.metadata.start, lts_payload.metadata.end - - -class _yaml_file_get: - def __init__(self, filename, yaml_file): - self.yaml_file = yaml_file - - def get(self, key, missing=...): - jsonpath_expression = jsonpath_ng.parse(f"$.{key}") - - match = jsonpath_expression.find(self.yaml_file) - if not match: - if missing != ...: - return missing - - raise KeyError(f"Key '{key}' not found in {self.yaml_file} ...") - - return match[0].value - - -def get_yaml_get_key(filename, yaml_file): - return _yaml_file_get(filename, yaml_file).get - - -def validate_lts_payload(lts_payload_model, payload, import_settings, reraise=False): - try: - json_lts = json.dumps( - payload, indent=4, default=functools.partial(json_dumper, strict=False) - ) - except ValueError as e: - logging.error(f"Couldn't dump the lts_payload into JSON :/ {e}") - - if reraise: - raise - - return False - - parsed_lts = json.loads(json_lts) - try: - lts_payload_model.parse_obj(parsed_lts) - return True - - except Exception as e: - log_fct = logging.error if reraise else logging.warning - log_fct("lts-error: Failed to validate the generated LTS payload against the model") - log_fct(f"lts-error: entry settings: {import_settings}") - - if reraise: - raise - - log_fct(f"lts-error: validation issue(s): {e}") - - return False diff --git a/projects/matrix_benchmarking/visualizations/helpers/store/k8s_quantity.py b/projects/matrix_benchmarking/visualizations/helpers/store/k8s_quantity.py deleted file mode 100644 index a4391dbb0..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/store/k8s_quantity.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Downloaded from -# https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/utils/quantity.py - -from decimal import Decimal, InvalidOperation - - -def parse_quantity(quantity): - """ - Parse kubernetes canonical form quantity like 200Mi to a decimal number. - Supported SI suffixes: - base1024: Ki | Mi | Gi | Ti | Pi | Ei - base1000: n | u | m | "" | k | M | G | T | P | E - - See https://github.com/kubernetes/apimachinery/blob/master/pkg/api/resource/quantity.go - - Input: - quantity: string. kubernetes canonical form quantity - - Returns: - Decimal - - Raises: - ValueError on invalid or unknown input - """ - if isinstance(quantity, (int, float, Decimal)): - return Decimal(quantity) - - exponents = {"n": -3, "u": -2, "m": -1, "K": 1, "k": 1, "M": 2, "G": 3, "T": 4, "P": 5, "E": 6} - - quantity = str(quantity) - number = quantity - suffix = None - if len(quantity) >= 2 and quantity[-1] == "i": - if quantity[-2] in exponents: - number = quantity[:-2] - suffix = quantity[-2:] - elif len(quantity) >= 1 and quantity[-1] in exponents: - number = quantity[:-1] - suffix = quantity[-1:] - - try: - number = Decimal(number) - except InvalidOperation: - raise ValueError(f"Invalid number format: {number}") - - if suffix is None: - return number - - if suffix.endswith("i"): - base = 1024 - elif len(suffix) == 1: - base = 1000 - else: - raise ValueError(f"{quantity} has unknown suffix") - - # handly SI inconsistency - if suffix == "ki": - raise ValueError(f"{quantity} has unknown suffix") - - if suffix[0] not in exponents: - raise ValueError(f"{quantity} has unknown suffix") - - exponent = Decimal(exponents[suffix[0]]) - return number * (base**exponent) diff --git a/projects/matrix_benchmarking/visualizations/helpers/store/parsers.py b/projects/matrix_benchmarking/visualizations/helpers/store/parsers.py deleted file mode 100644 index 76bcb9011..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/store/parsers.py +++ /dev/null @@ -1,429 +0,0 @@ -import datetime -import json -import logging -import os -import pathlib -import types -import urllib -import uuid -from functools import reduce - -import matrix_benchmarking.cli_args as cli_args -import matrix_benchmarking.store.prom_db as store_prom_db -import yaml - -import projects.matrix_benchmarking.visualizations.helpers.store as helpers_store - -from . import k8s_quantity - -K8S_TIME_FMT = "%Y-%m-%dT%H:%M:%SZ" -K8S_TIME_MILLI_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" -SHELL_DATE_TIME_FMT = "%a %b %d %H:%M:%S %Z %Y" -ANSIBLE_LOG_DATE_TIME_FMT = "%Y-%m-%d %H:%M:%S" - -register_important_file = lambda dirname, filename: dirname / filename - - -def ignore_file_not_found(fn): - def decorator(*args, **kwargs): - try: - return fn(*args, **kwargs) - except FileNotFoundError as e: - logging.warning(f"{fn.__name__}: FileNotFoundError: {e}") - return None - - return decorator - - -def dict_get_from_path(dict_obj, path, default=None): - # default currently unused :/ - return reduce(dict.get, path.split("."), dict_obj) - - -@ignore_file_not_found -def parse_env(dirname, test_config, capture_state_dir): - from_env = types.SimpleNamespace() - - ansible_env = {} - - from_env.test = types.SimpleNamespace() - from_env.test.run_id = None - from_env.test.test_path = None - from_env.test.ci_engine = None - from_env.test.urls = {} - - if not capture_state_dir: - logging.warning("no capture_state_dir received. Cannot parse the execution environment.") - return from_env - - with open(register_important_file(dirname, capture_state_dir / "_ansible.env")) as f: - for line in f.readlines(): - k, _, v = line.strip().partition("=") - - ansible_env[k] = v - - # --- - # eg 003__notebook_performance/003__sutest_notebooks__benchmark_performance/ - current_artifact_dir = pathlib.Path(ansible_env["ARTIFACT_DIR"]) - base_artifact_dir = "/logs/artifacts" - - if dirname.name == "from_url": - with open(dirname / "source_url") as f: # not an important file - source_url = f.read().strip() - _prefix, _, from_env.test.test_path = source_url.partition( - f"{from_env.test.run_id}/artifacts/" - ) - pass - else: - main_artifact_dir = pathlib.Path(ansible_env["TOPSAIL_FROM_CONFIG_FILE"]).parent - current_artifact_dir = pathlib.Path(ansible_env["ARTIFACT_DIR"]) - from_env.test.test_path = current_artifact_dir.relative_to(main_artifact_dir) - - if ansible_env.get("TOPSAIL_LOCAL_CI") == "true": - from_env.test.ci_engine = "TOPSAIL_LOCAL_CI" - from_env.test.run_id = ansible_env.get("TEST_RUN_IDENTIFIER") - - bucket_name = ansible_env.get("TOPSAIL_LOCAL_CI_BUCKET_NAME") - job_name_safe = ansible_env.get("JOB_NAME_SAFE") - - from_env.test.urls |= dict( - LOCAL_CI_S3=f"https://{bucket_name}.s3.amazonaws.com/index.html#local-ci/{job_name_safe}/{from_env.test.run_id}/{from_env.test.test_path}/", - ) - - if ansible_env.get("OPENSHIFT_CI") == "truez": - from_env.test.ci_engine = "OPENSHIFT_CI" - job_spec = json.loads(ansible_env["JOB_SPEC"].strip("'")) - entrypoint_options = json.loads(ansible_env["ENTRYPOINT_OPTIONS"]) - - job = job_spec["job"] - build_id = job_spec["buildid"] - - test_name = ansible_env.get("JOB_NAME_SAFE") - step_dir = ansible_env.get("TOPSAIL_OPENSHIFT_CI_STEP_DIR") - - # --- - # eg: pull/openshift-psap_topsail/181/pull-ci-openshift-psap-topsail-main-rhoai-light/1749833488137195520 - if job_spec["type"] == "periodic": - path = "logs" - - from_env.test.run_id = f"{job}/{build_id}/{test_name}" - - from_env.test.urls |= dict( - PROW_JOB=f"https://prow.ci.openshift.org/view/gs/test-platform-results/{path}/{job}/{build_id}/", - PROW_ARTIFACTS=f"https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/{path}/{job}/{build_id}/artifacts/{test_name}/{step_dir}/{from_env.test.test_path}", - ) - else: - pull_number = job_spec["refs"]["pulls"][0]["number"] - github_org = job_spec["refs"]["org"] - github_repo = job_spec["refs"]["repo"] - path = "pr-logs/pull" - from_env.test.run_id = f"{pull_number}/{job}/{build_id}/{test_name}" - - # --- - from_env.test.urls |= dict( - PROW_JOB=f"https://prow.ci.openshift.org/view/gs/test-platform-results/{path}/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}/", - PROW_ARTIFACTS=f"https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/{path}/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}/artifacts/{test_name}/{step_dir}/{from_env.test.test_path}", - ) - - if ansible_env.get("PERFLAB_CI") == "true": - from_env.test.ci_engine = "PERFLAB_CI" - - jumphost = ansible_env["JENKINS_JUMPHOST"] - build_number = ansible_env["JENKINS_BUILD_NUMBER"] - jenkins_job = ansible_env["JENKINS_JOB"] # "job/ExternalTeams/job/RHODS/job/topsail" - jenkins_instance = ansible_env["JENKINS_INSTANCE"] # ci.app-svc-perf.corp.redhat.com - - from_env.test.run_id = build_number - - from_env.test.urls |= dict( - JENKINS_ARTIFACTS=f"https://{jenkins_instance}/{jenkins_job}/{build_number}/artifact/run/{jumphost}/{from_env.test.test_path}" - ) - - if test_config and test_config.get("export_artifacts.enabled", False): - bucket = test_config.get("export_artifacts.bucket") - path_prefix = test_config.get("export_artifacts.path_prefix") - - if ansible_env.get("OPENSHIFT_CI") == "true": - job_spec = json.loads(ansible_env["JOB_SPEC"]) - pull_number = job_spec["refs"]["pulls"][0]["number"] - github_org = job_spec["refs"]["org"] - github_repo = job_spec["refs"]["repo"] - job = job_spec["job"] - build_id = job_spec["buildid"] - - s3_path = f"prow/{pull_number}/{build_id}/test_ci" - - elif ansible_env.get("PERFLAB_CI") == "true": - if os.environ.get("PERFLAB_CI") == "true": - build_number = os.environ["JENKINS_BUILD_NUMBER"] - job = os.environ["JENKINS_JOB"] # "job/ExternalTeams/job/RHODS/job/topsail" - else: - build_number = "---" - job = "job/ExternalTeams/job/RHODS/job/topsail" - - job_id = job[4:].replace("/job/", "_") - - s3_path = f"middleware_jenkins/{job_id}/{build_number}" - - elif ansible_env.get("TOPSAIL_LOCAL_CI") == "true": - bucket = ansible_env.get("TOPSAIL_LOCAL_CI_BUCKET_NAME") - job_name_safe = ansible_env.get("JOB_NAME_SAFE") - run_id = ansible_env.get("TEST_RUN_IDENTIFIER") - - path_prefix = "local-ci" - s3_path = f"{job_name_safe}/{run_id}" - - else: - raise RuntimeError("CI environment not recognized :/") - - from_env.test.urls |= dict( - RHOAI_CPT_S3=f"https://{bucket}.s3.amazonaws.com/index.html#{path_prefix}/{s3_path}/{from_env.test.test_path}/" - ) - - return from_env - - -@ignore_file_not_found -def parse_test_uuid(dirname): - with open(dirname / ".uuid") as f: - test_uuid = f.read().strip() - - return uuid.UUID(test_uuid) - - -@ignore_file_not_found -def parse_nodes_info(dirname, capture_state_dir, sutest_cluster=True): - nodes_info = {} - - if not capture_state_dir: - logging.warning("no capture_state_dir received. Cannot parse the node information.") - return nodes_info - - nodes_file = capture_state_dir / "nodes.json" - with open(register_important_file(dirname, nodes_file)) as f: - nodeList = json.load(f) - - if not nodeList["items"]: - logging.error(f"No nodes found in {nodes_file} ...") - return None - - for node in nodeList["items"]: - node_name = node["metadata"]["name"] - node_info = nodes_info[node_name] = types.SimpleNamespace() - - node_info.name = node_name - node_info.sutest_cluster = sutest_cluster - node_info.managed = "managed.openshift.com/customlabels" in node["metadata"]["annotations"] - node_info.instance_type = node["metadata"]["labels"].get( - "node.kubernetes.io/instance-type", "N/A" - ) - - node_info.control_plane = ( - "node-role.kubernetes.io/control-plane" in node["metadata"]["labels"] - or "node-role.kubernetes.io/master" in node["metadata"]["labels"] - ) - - node_info.infra = not node_info.control_plane - - if node["metadata"]["labels"].get("nvidia.com/gpu.present"): - product = node["metadata"]["labels"].get("nvidia.com/gpu.product") - if not product: - node_info.gpu = None - continue - - node_info.gpu = types.SimpleNamespace() - - node_info.gpu.product = product - if node["metadata"]["labels"].get("nvidia.com/gpu.memory"): - node_info.gpu.memory = ( - int(node["metadata"]["labels"].get("nvidia.com/gpu.memory")) / 1000 - ) - if node["metadata"]["labels"].get("nvidia.com/gpu.count"): - node_info.gpu.count = int(node["metadata"]["labels"].get("nvidia.com/gpu.count")) - - else: - node_info.gpu = None - - node_info.allocatable = types.SimpleNamespace() - node_info.allocatable.memory = float( - k8s_quantity.parse_quantity(node["status"]["allocatable"]["memory"]) - ) - node_info.allocatable.memory = float( - k8s_quantity.parse_quantity(node["status"]["allocatable"]["memory"]) - ) - node_info.allocatable.cpu = float( - k8s_quantity.parse_quantity(node["status"]["allocatable"]["cpu"]) - ) - - node_info.allocatable.gpu = int(node["status"]["allocatable"].get("nvidia.com/gpu", 0)) - node_info.allocatable.__dict__["nvidia.com/gpu"] = node_info.allocatable.gpu - - return nodes_info - - -def extract_cluster_info(nodes_info): - cluster_info = types.SimpleNamespace() - - cluster_info.node_count = [ - node_info for node_info in nodes_info.values() if node_info.sutest_cluster - ] - - cluster_info.control_plane = [ - node_info - for node_info in nodes_info.values() - if node_info.sutest_cluster and node_info.control_plane - ] - - cluster_info.infra = [ - node_info - for node_info in nodes_info.values() - if node_info.sutest_cluster and node_info.infra - ] - - cluster_info.gpus = [] - for infra_node in cluster_info.node_count: - if not infra_node.gpu: - continue - cluster_info.gpus.append(infra_node.gpu) - - return cluster_info - - -@ignore_file_not_found -def parse_rhods_info(dirname, capture_state_dir, version_name=None): - rhods_info = types.SimpleNamespace() - - if capture_state_dir is None: - logging.error( - "parse_rhods_info: `capture_state_dir` not available, returning dummy values :/" - ) - rhods_info.version = "not available" - rhods_info.createdAt_raw = "not available" - rhods_info.full_version = "0.0.0" - return rhods_info - - with open(register_important_file(dirname, capture_state_dir / "rhods.version")) as f: - rhods_info.version = f.read().strip() - - with open(register_important_file(dirname, capture_state_dir / "rhods.createdAt")) as f: - rhods_info.createdAt_raw = f.read().strip() - - try: - rhods_info.createdAt = datetime.datetime.strptime(rhods_info.createdAt_raw, K8S_TIME_FMT) - except ValueError: - logging.error("Couldn't parse RHOAI version timestamp: {e}") - rhods_info.createdAt = None - - if version_name: - rhods_info.full_version = ( - f"{rhods_info.version}-{version_name}+{rhods_info.createdAt.strftime('%Y-%m-%d')}" - ) - else: - logging.info("parse_rhods_info: no version_name provided.") - rhods_info.full_version = None - - return rhods_info - - -@ignore_file_not_found -def parse_ocp_version(dirname, capture_state_dir): - if not capture_state_dir: - logging.warning("no capture_state_dir received. Cannot parse OCP version") - return "" - - ocp_version_files = list((dirname / capture_state_dir).glob("ocp_version.y*ml")) - - if not ocp_version_files: - raise FileNotFoundError(f"no 'ocp_version.y*ml' in {dirname / capture_state_dir}") - ocp_version_file = ocp_version_files[0] - - with open(register_important_file(dirname, capture_state_dir / ocp_version_file.name)) as f: - sutest_ocp_version_yaml = yaml.safe_load(f) - - return sutest_ocp_version_yaml["openshiftVersion"] - - -def parse_test_config(dirname): - test_config = types.SimpleNamespace() - - filename = pathlib.Path("config.yaml") - test_config.filepath = dirname / filename - - with open(register_important_file(dirname, filename)) as f: - yaml_file = test_config.yaml_file = yaml.safe_load(f) - - if not yaml_file: - logging.error(f"Config file '{filename}' is empty ...") - yaml_file = test_config.yaml_file = {} - - test_config.name = f"topsail config ({test_config.filepath})" - test_config.get = helpers_store.get_yaml_get_key(test_config.name, yaml_file) - - return test_config - - -def parse_local_env(dirname): - from_local_env = types.SimpleNamespace() - - from_local_env.artifacts_basedir = None - from_local_env.source_url = None - from_local_env.is_interactive = False - - try: - with open(dirname / "source_url") as f: # not an important file - from_local_env.source_url = f.read().strip() - from_local_env.artifacts_basedir = pathlib.Path( - urllib.parse.urlparse(from_local_env.source_url).path - ) - except FileNotFoundError: - pass - - logging.debug( - f"Source_url: {from_local_env.source_url}, artifacts_basedir: {from_local_env.artifacts_basedir}" - ) - - if not cli_args.kwargs.get("generate"): - # running in interactive mode - from_local_env.is_interactive = True - from_local_env.artifacts_basedir = dirname - return from_local_env - - # running in generate mode - - # This must be parsed from the process env (not the file), to - # properly generate the error report links to the image. - job_name = os.getenv("JOB_NAME_SAFE", "") - - if job_name.endswith("-plot"): - # running independently of the test, the source_url file must be available - if from_local_env.source_url is None: - logging.warning(f"The source URL should be available when running from '{job_name}'") - from_local_env.source_url = "/missing/source/url" - from_local_env.artifacts_basedir = dirname - - elif "ARTIFACT_DIR" in os.environ: - # os.path.relpath can return '..', but pathlib.Path().relative_to cannot - from_local_env.source_url = pathlib.Path( - os.path.relpath(dirname, pathlib.Path(os.environ["ARTIFACT_DIR"])) - ) - from_local_env.artifacts_basedir = from_local_env.source_url - - else: - from_local_env.artifacts_basedir = dirname.absolute() - - return from_local_env - - -def extract_metrics(dirname, db_files): - metrics = {} - for name, (tarball_glob, metric) in db_files.items(): - try: - prom_tarball = list(dirname.glob(tarball_glob))[0] - except IndexError: - logging.warning(f"No {tarball_glob} in '{dirname}'.") - continue - - register_important_file(dirname, prom_tarball.relative_to(dirname)) - metrics[name] = store_prom_db.extract_metrics(prom_tarball, metric, dirname) - - return metrics diff --git a/projects/matrix_benchmarking/visualizations/helpers/store/prom.py b/projects/matrix_benchmarking/visualizations/helpers/store/prom.py deleted file mode 100644 index 53ac965af..000000000 --- a/projects/matrix_benchmarking/visualizations/helpers/store/prom.py +++ /dev/null @@ -1,719 +0,0 @@ -import projects.matrix_benchmarking.visualizations.helpers.plotting.prom as plotting_prom -import projects.matrix_benchmarking.visualizations.helpers.plotting.prom.cpu_memory as plotting_prom_cpu_memory - - -def get_metrics(name): - def _get_metrics(entry, metric): - try: - return entry.results.metrics[name][metric] - except KeyError: - return [] - - return _get_metrics - - -def _labels_to_string(labels, exclude=[]): - values = [] - for k, vals in labels.items(): - if k in exclude: - continue - - if not isinstance(vals, list): - vals = [vals] - - for v in vals: - if v.startswith("!~"): - op = "!~" - v = v.replace("!~", "") - else: - op = "=~" - - values.append(f"{k}{op}'{v}'") - - return ",".join(values) - - -def _get_container_cpu_queries(cluster_role, labels): - labels_str = _labels_to_string(labels) - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items()) - - return [ - { - f"{cluster_role}__container_cpu__{metric_name}": "rate(container_cpu_usage_seconds_total{" - + labels_str - + "}[5m])" - }, - { - f"{cluster_role}__container_sum_cpu__{metric_name}": "sum(rate(container_cpu_usage_seconds_total{" - + labels_str - + "}[5m]))" - }, - { - f"{cluster_role}__container_cpu_requests__{metric_name}": "kube_pod_container_resource_requests{" - + labels_str - + ",resource='cpu'}" - }, - { - f"{cluster_role}__container_cpu_limits__{metric_name}": "kube_pod_container_resource_limits{" - + labels_str - + ",resource='cpu'}" - }, - ] - - -def _get_container_mem_queries(cluster_role, labels): - labels_str = _labels_to_string(labels) - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items()) - - return [ - { - f"{cluster_role}__container_memory_working_set_bytes__{metric_name}": "container_memory_working_set_bytes{" - + labels_str - + "}" - }, - { - f"{cluster_role}__container_memory_usage_bytes__{metric_name}": "container_memory_usage_bytes{" - + labels_str - + "}" - }, - { - f"{cluster_role}__container_memory_rss__{metric_name}": "container_memory_rss{" - + labels_str - + "}" - }, - { - f"{cluster_role}__container_memory_requests__{metric_name}": "kube_pod_container_resource_requests{" - + labels_str - + ",resource='memory'}" - }, - { - f"{cluster_role}__container_memory_limits__{metric_name}": "kube_pod_container_resource_limits{" - + labels_str - + ",resource='memory'}" - }, - { - f"{cluster_role}__container_max_memory__{metric_name}": "container_memory_max_usage_bytes{" - + labels_str - + "}" - }, - ] - - -def _get_cluster_mem_queries(cluster_role): - return [ - {f"{cluster_role}__cluster_memory_capacity": "sum(cluster:capacity_memory_bytes:sum)"}, - { - f"{cluster_role}__cluster_memory_usage": """ sum( - (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) - * - on(instance) group_left(role) ( - label_replace(max by (node) (kube_node_role{role=~".+"}), "instance", "$1", "node", "(.*)") - ) - ) -""" - }, - { - f"{cluster_role}__cluster_memory_requests": """ sum( - kube_pod_resource_request{resource="memory"} - * - on(node) group_left(role) ( - max by (node) (kube_node_role{role=~".+"}) - ) - ) -""" - }, - ] - - -def _get_cluster_cpu_queries(cluster_role): - return [ - { - f"{cluster_role}__cluster_cpu_requests": """ sum( - kube_pod_resource_request{resource="cpu"} - * - on(node) group_left(role) ( - max by (node) (kube_node_role{role=~".+"}) - ) - ) -""" - }, - { - f"{cluster_role}__cluster_cpu_usage": """ sum( - ( - 1 - rate(node_cpu_seconds_total{mode="idle"}[2m]) - * - on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{pod=~"node-exporter.+"} - ) - * - on(node) group_left(role) ( - max by (node) (kube_node_role{role=~".+"}) - ) - ) -""" - }, - {f"{cluster_role}__cluster_cpu_capacity": "sum(cluster:capacity_cpu_cores:sum)"}, - ] - - -def _get_pod_disk_rate_queries(cluster_role, labels): - labels_str = _labels_to_string(labels, exclude="container") - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items() if k != "container") - - return [ - { - f"{cluster_role}__container_fs_reads_bytes_irate{metric_name}": "(sum(irate(container_fs_reads_bytes_total{" - + labels_str - + "}[5m])) by (pod, namespace))" - }, - { - f"{cluster_role}__container_fs_writes_bytes_irate{metric_name}": "(sum(irate(container_fs_writes_bytes_total{" - + labels_str - + "}[5m])) by (pod, namespace))" - }, - ] - - -def _get_pod_disk_total_queries(cluster_role, labels): - labels_str = _labels_to_string(labels, exclude="container") - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items() if k != "container") - - return [ - { - f"{cluster_role}__container_fs_reads_bytes_total{metric_name}": "(sum(container_fs_reads_bytes_total{" - + labels_str - + "})) by (pod, namespace))" - }, - { - f"{cluster_role}__container_fs_writes_bytes_total{metric_name}": "(sum(container_fs_writes_bytes_total{" - + labels_str - + "}) by (pod, namespace))" - }, - ] - - -def _get_pod_network_rate_queries(cluster_role, labels): - labels_str = _labels_to_string(labels, exclude="container") - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items() if k != "container") - - return [ - { - f"{cluster_role}__container_network_receive_bytes_irate__{metric_name}": "(sum(irate(container_network_receive_bytes_total{" - + labels_str - + "}[5m])) by (pod, namespace, interface)) + on(namespace,pod,interface) group_left(network_name) (pod_network_name_info)" - }, - { - f"{cluster_role}__container_network_transmit_bytes_irate__{metric_name}": "(sum(irate(container_network_transmit_bytes_total{" - + labels_str - + "}[5m])) by (pod, namespace, interface)) + on(namespace,pod,interface) group_left(network_name) (pod_network_name_info)" - }, - ] - - -def _get_pod_network_total_queries(cluster_role, labels): - labels_str = _labels_to_string(labels, exclude="container") - - metric_name = "_".join(f"{k}={v}" for k, v in labels.items() if k != "container") - - return [ - { - f"{cluster_role}__container_network_receive_bytes_total__{metric_name}": "(sum(container_network_receive_bytes_total{" - + labels_str - + "}) by (pod, namespace, interface)) + on(namespace,pod,interface) group_left(network_name) (pod_network_name_info)" - }, - { - f"{cluster_role}__container_network_transmit_bytes_total__{metric_name}": "(sum(container_network_transmit_bytes_total{" - + labels_str - + "}) by (pod, namespace, interface)) + on(namespace,pod,interface) group_left(network_name) (pod_network_name_info)" - }, - ] - - -# --- - - -def _get_cluster_mem_cpu(cluster_role, register): - all_metrics = [] - - cluster_mem = _get_cluster_mem_queries(cluster_role) - cluster_cpu = _get_cluster_cpu_queries(cluster_role) - - all_metrics += cluster_mem - all_metrics += cluster_cpu - - if register: - plotting_prom_cpu_memory.Plot( - cluster_mem, - f"{cluster_role} cluster memory usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - is_cluster=True, - is_memory=True, - ) - - plotting_prom_cpu_memory.Plot( - cluster_cpu, - f"{cluster_role} cluster CPU usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - is_cluster=True, - ) - - return all_metrics - - -def _get_container_mem_cpu_metrics(cluster_role, register, label_sets): - all_metrics = [] - - for plot_name_labels in label_sets: - plot_name, labels = list(plot_name_labels.items())[0] - - mem = _get_container_mem_queries(cluster_role, labels) - cpu = _get_container_cpu_queries(cluster_role, labels) - - all_metrics += mem - all_metrics += cpu - - if not register: - continue - - container = labels.get("container", "all") - - plotting_prom_cpu_memory.Plot( - cpu, - f"{plot_name}: CPU usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - container_name=container, - ) - plotting_prom_cpu_memory.Plot( - mem, - f"{plot_name}: Mem usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - is_memory=True, - ) - - return all_metrics - - -def _get_disk_usage_metrics(cluster_role, register, label_sets, disk_metrics_names): - all_metrics = [] - - for plot_name_labels in label_sets: - plot_name, labels = list(plot_name_labels.items())[0] - if plot_name not in disk_metrics_names: - continue - - disk_queries = _get_pod_disk_total_queries(cluster_role, labels) - - all_metrics += disk_queries - - if not register: - continue - - plotting_prom.Plot( - disk_queries, - f"Prom: {plot_name}: Disk total usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - title="Disk total usage", - y_title="in GB", - y_divisor=1000 * 1000 * 1000, - substract_first=True, - ) - - for plot_name_labels in label_sets: - plot_name, labels = list(plot_name_labels.items())[0] - if plot_name not in disk_metrics_names: - continue - - disk_queries = _get_pod_disk_rate_queries(cluster_role, labels) - - all_metrics += disk_queries - - if not register: - continue - - plotting_prom.Plot( - disk_queries, - f"Prom: {plot_name}: Disk rate usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - title="Disk rate usage", - y_title="in MB/s", - y_divisor=1000 * 1000, - ) - - return all_metrics - - -def _get_network_usage_metrics(cluster_role, register, label_sets, network_metrics_names): - all_metrics = [] - - for plot_name_labels in label_sets: - plot_name, labels = list(plot_name_labels.items())[0] - if plot_name not in network_metrics_names: - continue - - network_queries = _get_pod_network_total_queries(cluster_role, labels) - all_metrics += network_queries - - if not register: - continue - - plotting_prom.Plot( - network_queries, - f"Prom: {plot_name}: Network total usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - title="Network total usage", - y_title="in GB", - y_divisor=1000 * 1000 * 1000, - substract_first=True, - ) - - for plot_name_labels in label_sets: - plot_name, labels = list(plot_name_labels.items())[0] - if plot_name not in network_metrics_names: - continue - - network_queries = _get_pod_network_rate_queries(cluster_role, labels) - all_metrics += network_queries - - if not register: - continue - - plotting_prom.Plot( - network_queries, - f"Prom: {plot_name}: Network rate usage", - get_metrics=get_metrics(cluster_role), - as_timestamp=True, - title="Network rate usage", - y_title="in MBps", - y_divisor=1000 * 1000, - ) - - return all_metrics - - -def _get_control_plane_nodes_cpu_usage(cluster_role, register): - all_metrics = [ - { - f"{cluster_role.title()} Control Plane Node CPU usage": "sum(irate(node_cpu_seconds_total[2m])) by (mode, instance) " - }, - { - f"{cluster_role.title()} Control Plane Node CPU idle": 'sum(irate(node_cpu_seconds_total{mode="idle"}[2m])) by (mode, instance) ' - }, - ] - - def get_legend_name(metric_name, metric_metric): - name = metric_metric["mode"] + " | " + metric_metric["instance"].split(".")[0] - group = metric_metric["instance"].split(".")[0] - return name, group - - def only_control_plane_nodes(entry, metrics): - control_plane_nodes = [ - node.name - for node in ( - entry.results.cluster_info.control_plane - if getattr(entry.results, "cluster_info", []) - else [] - ) - ] - - for metric in metrics: - if metric.metric["instance"] not in control_plane_nodes: - continue - yield metric - - def no_control_plane_nodes(entry, metrics): - control_plane_nodes = [ - node.name - for node in ( - entry.results.cluster_info.control_plane - if getattr(entry.results, "cluster_info", []) - else [] - ) - ] - - for metric in metrics: - if metric.metric["instance"] in control_plane_nodes: - continue - yield metric - - if register: - for metric in all_metrics: - name, rq = list(metric.items())[0] - - plotting_prom.Plot( - {name: rq}, - f"Prom: {name}", - None, - "Count", - get_metrics=get_metrics(cluster_role), - filter_metrics=only_control_plane_nodes, - get_legend_name=get_legend_name, - show_queries_in_title=True, - show_legend=True, - higher_better=True if "CPU idle" in name else False, - as_timestamp=True, - ) - - plotting_prom.Plot( - {name: rq}, - f"Prom: {name}".replace("Control Plane", "Worker"), - None, - "Count", - get_metrics=get_metrics(cluster_role), - filter_metrics=no_control_plane_nodes, - get_legend_name=get_legend_name, - show_queries_in_title=True, - show_legend=True, - higher_better=True if "CPU idle" in name else False, - as_timestamp=True, - ) - - return all_metrics - - -def _get_control_plane_nodes(cluster_role, register): - all_metrics = [] - all_metrics += _get_container_mem_cpu_metrics( - cluster_role, - register, - [ - { - f"{cluster_role.title()} ApiServer": dict( - namespace="openshift-kube-apiserver", - pod=["!~kube-apiserver-guard.*", "kube-apiserver-.*"], - ) - } - ], - ) - all_metrics += _get_container_mem_cpu_metrics( - cluster_role, - register, - [ - { - f"{cluster_role.title()} ETCD": dict( - namespace="openshift-etcd", pod=["!~etcd-guard-.*", "etcd-.*"] - ) - } - ], - ) - - return all_metrics - - -def _get_apiserver_errcodes(cluster_role, register): - all_metrics = [] - - apiserver_request_metrics = [ - { - f"{cluster_role.title()} API Server Requests (successes)": 'sum by (code) (increase(apiserver_request_total{code=~"2.."}[2m]))' - }, - { - f"{cluster_role.title()} API Server Requests (client errors)": 'sum by (code) (increase(apiserver_request_total{code=~"4.."}[2m]))' - }, - { - f"{cluster_role.title()} API Server Requests (server errors)": 'sum by (code) (increase(apiserver_request_total{code=~"5.."}[2m]))' - }, - ] - - apiserver_request_duration_metrics = [ - { - f"{cluster_role.title()} API Server {verb} Requests duration": f'histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{{apiserver="kube-apiserver", verb="{verb}", subresource!="log"}}[2m])) by (resource,subresource,le)) > 0' - } - for verb in ["GET", "PUT", "LIST", "PATCH"] - ] - - all_metrics += apiserver_request_metrics - all_metrics += apiserver_request_duration_metrics - - def get_apiserver_request_metrics_legend_name(metric_name, metric_metric): - return f"code={metric_metric['code']}", None - - def get_apiserver_request_duration_metrics_legend_name(metric_name, metric_metric): - try: - res = metric_metric["resource"] - except KeyError: - return str(metric_metric), None - - if metric_metric.get("subresource"): - res += f"/{metric_metric['subresource']}" - return res, None - - if register: - for metric in apiserver_request_metrics: - name, rq = list(metric.items())[0] - - plotting_prom.Plot( - {name: rq}, - f"Prom: {name}", - None, - "Count", - get_metrics=get_metrics(cluster_role), - get_legend_name=get_apiserver_request_metrics_legend_name, - show_queries_in_title=True, - show_legend=True, - as_timestamp=True, - ) - - for metric in apiserver_request_duration_metrics: - name, rq = list(metric.items())[0] - plotting_prom.Plot( - {name: rq}, - f"Prom: {name}", - None, - "Count", - get_metrics=get_metrics(cluster_role), - get_legend_name=get_apiserver_request_duration_metrics_legend_name, - show_queries_in_title=True, - show_legend=True, - as_timestamp=True, - ) - - return all_metrics - - -def get_gpu_usage_metrics(cluster_role, register, container): - all_metrics = [] - - gpu_usage_metrics = [ - { - f"{cluster_role.title()} GPU memory used": f'DCGM_FI_DEV_FB_USED{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU memory used (all GPUs)": f'sum(DCGM_FI_DEV_FB_USED{{exported_container="{container}"}})' - }, - { - f"{cluster_role.title()} GPU active computes": f'DCGM_FI_PROF_SM_ACTIVE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU computes occupancy": f'DCGM_FI_PROF_SM_OCCUPANCY{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU memory transfer utilization": f'DCGM_FI_DEV_MEM_COPY_UTIL{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU memory unallocated": f'DCGM_FI_DEV_FB_FREE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU compute utilization (not 100% accurate)": f'DCGM_FI_DEV_GPU_UTIL{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU engine usage (not 100% accurate)": f'DCGM_FI_PROF_GR_ENGINE_ACTIVE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU active fp16 pipe": f'DCGM_FI_PROF_PIPE_FP16_ACTIVE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU active fp32 pipe": f'DCGM_FI_PROF_PIPE_FP32_ACTIVE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU active fp64 pipe": f'DCGM_FI_PROF_PIPE_FP64_ACTIVE{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU NVLink transfer (rx)": f'DCGM_FI_PROF_NVLINK_RX_BYTES{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU NVLink transfer (tx)": f'DCGM_FI_PROF_NVLINK_TX_BYTES{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU PCIe transfer (rx)": f'DCGM_FI_PROF_PCIE_RX_BYTES{{exported_container="{container}"}}' - }, - { - f"{cluster_role.title()} GPU PCIe transfer (tx)": f'DCGM_FI_PROF_PCIE_TX_BYTES{{exported_container="{container}"}}' - }, - ] - all_metrics += gpu_usage_metrics - - def get_legend_name(metric_name, metric_metric): - name = metric_metric.get("Hostname", "") - - return name, None - - if register: - for metric in gpu_usage_metrics: - name, rq = list(metric.items())[0] - - y_divisor = 1 - - if "DCGM_FI_DEV_FB_" in rq: - y_title = f"{name} (in GiB)" - y_divisor = 1024 - - elif "DCGM_FI_PROF_PCIE_" in rq or "DCGM_FI_PROF_NVLINK" in rq: - y_title = "Data transfer (in MiB/s)" - y_divisor = 1024 - elif "DCGM_FI_DEV_GPU_UTIL" in rq or "DCGM_FI_PROF_GR_ENGINE_ACTIVE" in rq: - y_title = "Compute usage (in %)" - if "DCGM_FI_PROF_GR_ENGINE_ACTIVE" in rq: - y_divisor = 0.01 - elif "DCGM_FI_DEV_MEM_COPY_UTIL" in rq: - y_title = "GPU transfer bus usage (in %)" - elif "DCGM_FI_PROF_SM_ACTIVE" in rq: - y_title = "The ratio of cycles an SM has at least 1 warp assigned (in %)" - y_divisor = 0.01 - elif "DCGM_FI_PROF_SM_OCCUPANCY" in rq: - y_title = "The ratio of number of warps resident on an SM (in %)" - y_divisor = 0.01 - elif "DCGM_FI_PROF_PIPE_FP" in rq: - y_title = "Ratio of cycles the fp pipes are active (in %)" - y_divisor = 0.01 - else: - y_title = "(no name)" - - plotting_prom.Plot( - {name: rq}, - f"Prom: {name}", - None, - y_title, - get_metrics=get_metrics(cluster_role), - get_legend_name=get_legend_name, - show_queries_in_title=True, - y_divisor=y_divisor, - show_legend=True, - as_timestamp=True, - ) - - return all_metrics - - -def get_cluster_metrics( - cluster_role, - *, - container_labels=[], - gpu_container=False, - disk_metrics_names=[], - network_metrics_names=[], - register=False, -): - all_metrics = [] - all_metrics += _get_cluster_mem_cpu(cluster_role, register) - all_metrics += _get_control_plane_nodes(cluster_role, register) - all_metrics += _get_apiserver_errcodes(cluster_role, register) - all_metrics += _get_control_plane_nodes_cpu_usage(cluster_role, register) - - if container_labels: - all_metrics += _get_container_mem_cpu_metrics(cluster_role, register, container_labels) - - if gpu_container: - all_metrics += get_gpu_usage_metrics(cluster_role, register, container=gpu_container) - - if disk_metrics_names: - all_metrics += _get_disk_usage_metrics( - cluster_role, register, container_labels, disk_metrics_names - ) - - if network_metrics_names: - all_metrics += _get_network_usage_metrics( - cluster_role, register, container_labels, network_metrics_names - ) - - return all_metrics From 914233423b24f0ed37003e96d664659772cb16d3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:18:04 +0200 Subject: [PATCH 0877/1072] [projects/legacy] Remove --- bin/run_ansible_toolbox.py | 1 - projects/legacy/ansible-config/ansible.cfg | 39 -- .../callback_plugins/human_log.py | 200 ------- .../callback_plugins/json_to_logfile.py | 183 ------ .../legacy/ansible-config/inventory/hosts | 5 - projects/legacy/library/__init__.py | 23 - projects/legacy/library/ansible_toolbox.py | 403 ------------- projects/legacy/library/common.py | 25 - projects/legacy/library/config.py | 528 ------------------ projects/legacy/library/env.py | 117 ---- projects/legacy/library/export.py | 118 ---- projects/legacy/library/run.py | 309 ---------- projects/legacy/library/run_toolbox.py | 42 -- projects/legacy/library/sizing.machines | 69 --- projects/legacy/library/sizing.py | 144 ----- 15 files changed, 2206 deletions(-) delete mode 120000 bin/run_ansible_toolbox.py delete mode 100644 projects/legacy/ansible-config/ansible.cfg delete mode 100644 projects/legacy/ansible-config/callback_plugins/human_log.py delete mode 100644 projects/legacy/ansible-config/callback_plugins/json_to_logfile.py delete mode 100644 projects/legacy/ansible-config/inventory/hosts delete mode 100644 projects/legacy/library/__init__.py delete mode 100644 projects/legacy/library/ansible_toolbox.py delete mode 100644 projects/legacy/library/common.py delete mode 100644 projects/legacy/library/config.py delete mode 100644 projects/legacy/library/env.py delete mode 100755 projects/legacy/library/export.py delete mode 100644 projects/legacy/library/run.py delete mode 100755 projects/legacy/library/run_toolbox.py delete mode 100644 projects/legacy/library/sizing.machines delete mode 100644 projects/legacy/library/sizing.py diff --git a/bin/run_ansible_toolbox.py b/bin/run_ansible_toolbox.py deleted file mode 120000 index 9c3341b91..000000000 --- a/bin/run_ansible_toolbox.py +++ /dev/null @@ -1 +0,0 @@ -../projects/legacy/library/run_toolbox.py \ No newline at end of file diff --git a/projects/legacy/ansible-config/ansible.cfg b/projects/legacy/ansible-config/ansible.cfg deleted file mode 100644 index aa6835e4f..000000000 --- a/projects/legacy/ansible-config/ansible.cfg +++ /dev/null @@ -1,39 +0,0 @@ -# config file for ansible -- http://ansible.com/ -# ============================================== - -# This config file provides examples for running -# the OpenShift playbooks with the provided -# inventory scripts. - -[defaults] - -forks = 20 -host_key_checking = False -retry_files_enabled = False -retry_files_save_path = /tmp/ansible-installer-retries -nocows = True -remote_user = root - -# Both roles_path and collections_paths are configured -# dynamically in the_common.py library, this is to avoid -# removing the default paths from the ansible interpreter. - -gathering = smart -callbacks_enabled = json_to_logfile, timer, profile_roles -inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt, .ini -# work around privilege escalation timeouts in ansible: -timeout = 30 - -stdout_callback = human_log -callback_plugins = callback_plugins -enable_task_debugger = False - -# Uncomment to use the provided example inventory -inventory = inventory/hosts - -# Make jinja less strict -jinja2_extensions = jinja2.ext.do - -[inventory] -# fail more helpfully when the inventory file does not parse (Ansible 2.4+) -unparsed_is_failed=true diff --git a/projects/legacy/ansible-config/callback_plugins/human_log.py b/projects/legacy/ansible-config/callback_plugins/human_log.py deleted file mode 100644 index bdfef0723..000000000 --- a/projects/legacy/ansible-config/callback_plugins/human_log.py +++ /dev/null @@ -1,200 +0,0 @@ -import os - -import ansible.executor - -# extends this class: https://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/callback/default.py -# mute this log message: -# "Friday 05 April 2024 15:01:14 +0200 (0:00:04.215) 0:00:04.504 **********" -import ansible_collections.ansible.posix.plugins.callback.profile_roles as profile_roles_mod -from ansible import constants as C -from ansible.plugins.callback.default import CallbackModule as default_CallbackModule - -profile_roles_mod.tasktime = lambda: "" - -INTERESTING_MODULE_PROPS = { - "stat": ["path", "exists", "mode"], - "invocation": None, - "_ansible_no_log": None, - "_ansible_verbose_always": None, - "__ansible_verbose_override": None, - "msg": None, - "changed": None, - "ansible_facts": None, - "include_args": None, - "failed_when_result": None, -} - - -class CallbackModule(default_CallbackModule): - def __display_result(self, result, color, ignore_errors=None, loop_idx=0): - if ignore_errors not in (None, False): - self._display.display(f"==> FAILED | ignore_errors={ignore_errors}", color=color) - - try: - if result._result["msg"].strip(): - self._display.display(f"msg: {result._result['msg']}", color=color) - except KeyError: - pass - - try: - if result._result.get("rc", 0) != 0: - self._display.display(f"return code: {result._result['rc']}", color=color) - except KeyError: - pass - - if "results" in result._result: - for idx, res in enumerate(result._result["results"]): - var = res["ansible_loop_var"] - value = res["_ansible_item_label"] - self._display.display("", color=color) - self._display.display(f"LOOP #{idx}: {var} = {value}", color=color) - item_result = ansible.executor.task_result.TaskResult( - host=result._host, - task=result._task, - return_data=res, - task_fields=result._task_fields, - ) - self.__display_result(item_result, color, ignore_errors, loop_idx=idx) - - return - - def print_result_as_dict(key, d, depth=1): - for k, v in d.items(): - if k in INTERESTING_MODULE_PROPS: - if INTERESTING_MODULE_PROPS[k] is None: - continue - if v not in INTERESTING_MODULE_PROPS[k]: - continue - - if not isinstance(v, dict): - self._display.display(f"{' ' * depth}- {k}:\t{v}", color=C.COLOR_VERBOSE) - else: - self._display.display(f"{' ' * depth}- {k}", color=C.COLOR_VERBOSE) - print_result_as_dict(k, v, depth + 1) - - try: - self.__print_cmd(result, color) - except KeyError: - print_result_as_dict("_top", result._result) - # ^^^ will print stdout and stderr - return - - #### - - self.__print_std_lines(result, "stdout", color) - self.__print_std_lines(result, "stderr", color) - - def __print_std_lines(self, result, std_name, color): - if not result._result.get(f"{std_name}_lines"): - return - - for line in result._result[f"{std_name}_lines"]: - self._display.display(f"<{std_name}> {line}", color=color) - - def __print_cmd(self, result, color): - cmd = result._result["cmd"] - - str_cmd = cmd if isinstance(cmd, str) else " ".join(cmd) - - self._display.display("", color=color) - if "\n" in str_cmd: - self._display.display(f"\n{str_cmd}\n", color=color) - else: - self._display.display(f" {str_cmd}", color=color) - - self._display.display("", color=color) - - def v2_runner_on_skipped(self, result): - self._print_task_banner(result._task, head=True) - - self._display.display( - f"==> SKIPPED | {result._result.get('skip_reason', '(no reason provided)')}", - color=C.COLOR_SKIP, - ) - for condition in result._task.when: - self._display.display(f"when: {condition}", C.COLOR_SKIP) - - def v2_runner_on_failed(self, result, ignore_errors=False): - self._print_task_banner(result._task, head=True) - - color = C.COLOR_VERBOSE if ignore_errors else C.COLOR_ERROR - self._display.display("----- FAILED ----", C.COLOR_ERROR) - self.__display_result(result, color, ignore_errors) - self._display.display("----- FAILED ----", C.COLOR_ERROR) - - def v2_runner_on_ok(self, result): - self._print_task_banner(result._task, head=True) - - color = C.COLOR_CHANGED if result._result.get("changed", False) else C.COLOR_OK - - self.__display_result(result, color) - - def v2_runner_on_unreachable(self, result): - del result._result[ - "unreachable" - ] # no need for `__display_result` to tell that, we already do it here - self._display.display(f"----- HOST UNREACHABLE ({result._host})----", C.COLOR_ERROR) - self.__display_result(result, C.COLOR_VERBOSE, False) - self._display.display("----- HOST UNREACHABLE ----", C.COLOR_ERROR) - - # items are handled as part of the 'normal' task logging - def v2_runner_item_on_failed(self, result): - pass - - def v2_runner_item_on_ok(self, result): - pass - - def v2_runner_item_on_skipped(self, result): - pass - - def _print_task_banner(self, task, head=False): - if head: - pass - - # self._display.display(f"{task}") - - else: - self._display.display("") - # followed by: - "Monday 08 March 2021 10:38:44 +0100 (0:00:00.023) 0:00:06.476 **********" - - def v2_runner_retry(self, result): - color = C.COLOR_VERBOSE - - if result._result["attempts"] == 1: - self._display.display(f"{result._task}", color=color) - self.__display_result(result, color) - self._display.display("") - - self._display.display("---") - self._display.display(str(result._task)) - self._display.display( - f"{result._task.get_path().replace(os.getcwd() + '/', '')}", color=color - ) - try: - self.__print_cmd(result, color) - except KeyError: - pass # ignore - - self.__print_std_lines(result, "stdout", color) - self.__print_std_lines(result, "stderr", color) - self._display.display( - f"==> failed attempt #{result._result['attempts']}/{result._task.retries}", - color=color, - ) - - self._display.display("") - - def v2_runner_on_start(self, host, task): - pass - - def v2_playbook_on_task_start(self, task, is_conditional): - self._display.display("") - self._display.display("~" * 79) - self._display.display(f"~~ {task.get_path().replace(os.getcwd() + '/', '')}") - self._display.display(f"~~ {task}") - - self._display.display("~" * 79) - - # followed by: - "Friday 05 April 2024 15:01:14 +0200 (0:00:04.215) 0:00:04.504 **********" diff --git a/projects/legacy/ansible-config/callback_plugins/json_to_logfile.py b/projects/legacy/ansible-config/callback_plugins/json_to_logfile.py deleted file mode 100644 index d3ebbc91b..000000000 --- a/projects/legacy/ansible-config/callback_plugins/json_to_logfile.py +++ /dev/null @@ -1,183 +0,0 @@ -# (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -# imported from /usr/lib/python3.8/site-packages/ansible/plugins/callback/syslog_json.py -# extending class from /usr/lib/python3.8/site-packages/ansible/plugins/callback/__init__.py - -# Make coding more python3-ish - - -import json -import socket - -from ansible.plugins.callback import CallbackBase - -DOCUMENTATION = """ - callback: json_to_logfile - callback_type: notification - requirements: - - whitelist in configuration - short_description: sends JSON events to a file - version_added: "1.9" - description: - - This plugin logs ansible-playbook and ansible runs to a file in then JSON format. - options: - logfile: - description: filename where the logs will be stored - env: - - name: ANSIBLE_JSON_TO_LOGFILE - default: /tmp/ansible.log.json - ini: - - section: callback_json_to_file - key: logfile -""" - - -class CallbackModule(CallbackBase): - """ - logs ansible playbook execution to a file in the JSON format - """ - - CALLBACK_VERSION = 1.0 - CALLBACK_TYPE = "aggregate" - CALLBACK_NAME = "json_to_file" - CALLBACK_NEEDS_WHITELIST = True - - def __init__(self): - super().__init__() - - def set_options(self, task_keys=None, var_options=None, direct=None): - super().set_options(task_keys=task_keys, var_options=var_options, direct=direct) - - self.logfile = self.get_option("logfile") - - with open(self.logfile, "a+") as f: - print("[", file=f) - self.is_open = True - - print("JSON_TO_LOGFILE: Storing json logs in", self.logfile) - self.hostname = socket.gethostname() - - def _write(self, data, finished=False): - self._warn_if_not_open() - - if not finished: - end = "," + "\n" - else: - end = "\n]" + "\n" - self.is_open = False - - with open(self.logfile, "a+") as f: - print(json.dumps(data, indent=4, sort_keys=True), end=end, file=f) - - def _warn_if_not_open(self): - if self.is_open: - return - - print("JSON_TO_LOGFILE: WARNING: logfile already closed ....") - - def playbook_on_stats(self, stats): - hosts = set() - for dictt in stats.ok, stats.failures, stats.skipped, stats.rescued: - hosts.update(dictt.keys()) - - all_stats = {} - for host in hosts: - all_stats[host] = stats.summarize(host) - - self._write( - { - "scope": "playbook", - "log_level": "info", - "status": "finished", - "stats": all_stats, - }, - finished=True, - ) - - def runner_on_failed(self, host, res, ignore_errors=False): - self._write( - { - "scope": "task", - "status": "FAILED", - "log_level": "error", - "host": host, - "hostname": self.hostname, - "ignore_errors": ignore_errors, - "message": res, - } - ) - - def runner_on_ok(self, host, res): - self._write( - { - "scope": "task", - "status": "OK", - "log_level": "info", - "host": host, - "hostname": self.hostname, - "message": res, - } - ) - - def runner_on_skipped(self, host, item=None): - self._write( - { - "scope": "task", - "status": "SKIPPED", - "log_level": "info", - "host": host, - "hostname": self.hostname, - "item": item, - "message": "skipped", - } - ) - - def runner_on_unreachable(self, host, res): - self._write( - { - "scope": "task", - "status": "UNREACHABLE", - "log_level": "error", - "host": host, - "hostname": self.hostname, - "message": res, - } - ) - - def runner_on_async_failed(self, host, res, jid): - self._write( - { - "scope": "task", - "status": "FAILED", - "log_level": "error", - "host": host, - "jid": jid, - "hostname": self.hostname, - "message": res, - } - ) - - def playbook_on_import_for_host(self, host, imported_file): - self._write( - { - "scope": "playbook", - "status": "IMPORTED", - "log_level": "error", - "host": host, - "hostname": self.hostname, - "file": imported_file, - } - ) - - def playbook_on_not_import_for_host(self, host, missing_file): - self._write( - { - "scope": "playbook", - "status": "NOT IMPORTED", - "log_level": "error", - "host": host, - "hostname": self.hostname, - "file": missing_file, - } - ) diff --git a/projects/legacy/ansible-config/inventory/hosts b/projects/legacy/ansible-config/inventory/hosts deleted file mode 100644 index cc3347580..000000000 --- a/projects/legacy/ansible-config/inventory/hosts +++ /dev/null @@ -1,5 +0,0 @@ -[all:vars] - -[targets] -localhost ansible_host=localhost - diff --git a/projects/legacy/library/__init__.py b/projects/legacy/library/__init__.py deleted file mode 100644 index ed937edf6..000000000 --- a/projects/legacy/library/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging - - -def configure_logging(): - logging.getLogger().setLevel(logging.INFO) - - logging.basicConfig( - format="%(asctime)s %(levelname)-4s [%(filename)-10s:%(lineno)3d] %(message)s", - datefmt="%H:%M:%S", - level=logging.INFO, - ) - - -def merge_dicts(a, b, path=None): - "updates a with b" - if path is None: - path = [] - for key in b: - if key in a and isinstance(a[key], dict) and isinstance(b[key], dict): - merge_dicts(a[key], b[key], path + [str(key)]) - else: - a[key] = b[key] - return a diff --git a/projects/legacy/library/ansible_toolbox.py b/projects/legacy/library/ansible_toolbox.py deleted file mode 100644 index 5e8fbfcf0..000000000 --- a/projects/legacy/library/ansible_toolbox.py +++ /dev/null @@ -1,403 +0,0 @@ -import functools -import importlib -import logging -import os -import pathlib -import shlex -import shutil -import subprocess -import sys -import tempfile -import time - -import yaml - -from projects.legacy.library import config - -TOPSAIL_DIR = pathlib.Path(config.__file__).parents[3] - -ANSIBLE_OS_CONFIGS_YAML = """ -windows: - - ansible_shell_type=powershell - - ansible_shell_executable=powershell.exe - - ansible_python_interpreter=auto_silent - - ansible_os_family=Windows - - ansible_become=no -darwin: - - ansible_os_family=Darwin -linux: - - ansible_os_family=Linux -""" -ANSIBLE_OS_CONFIGURATIONS = yaml.safe_load(ANSIBLE_OS_CONFIGS_YAML) - - -class Toolbox: - """ - The Topsail Toolbox - """ - - def __init__(self): - for toolbox_file in (TOPSAIL_DIR / "projects").glob("*/toolbox/*.py"): - if toolbox_file.name.startswith("."): - continue - - project_toolbox_module = str( - toolbox_file.relative_to(TOPSAIL_DIR).with_suffix("") - ).replace(os.path.sep, ".") - mod = importlib.import_module(project_toolbox_module) - toolbox_name = toolbox_file.with_suffix("").name - - if toolbox_name.startswith("_"): - continue - - if hasattr(mod, "__entrypoint"): - self.__dict__[toolbox_name] = getattr(mod, "__entrypoint") - continue - - try: - self.__dict__[toolbox_name] = getattr(mod, toolbox_name.title()) - except AttributeError as e: - logging.warning( - str(e) - ) # eg: AttributeError: module 'projects.notebooks.toolbox.notebooks' has no attribute 'Notebooks' - - -def AnsibleRole(role_name): - def decorator(fct): - fct.ansible_role = role_name - - @functools.wraps(fct) - def call_fct(*args, **kwargs): - run_ansible_role = fct(*args, **kwargs) - - run_ansible_role.role_name = role_name - run_ansible_role.ansible_constants = getattr(fct, "ansible_constants", {}) - run_ansible_role.ansible_mapped_params = getattr(fct, "ansible_mapped_params", False) - run_ansible_role.ansible_skip_config_generation = getattr( - fct, "ansible_skip_config_generation", False - ) - run_ansible_role.ansible_gather_facts = getattr(fct, "ansible_gather_facts", False) - - if not run_ansible_role.group: - run_ansible_role.group = fct.__qualname__.partition(".")[0].lower() - if not run_ansible_role.command: - run_ansible_role.command = fct.__name__ - - if not hasattr(run_ansible_role, "py_command_args"): - run_ansible_role.py_command_args = None - return run_ansible_role - - return call_fct - - return decorator - - -def AnsibleMappedParams(fct): - fct.ansible_mapped_params = True - return fct - - -def AnsibleSkipConfigGeneration(fct): - fct.ansible_skip_config_generation = True - return fct - - -def AnsibleConstant(description, name, value): - def decorator(fct): - if not hasattr(fct, "ansible_constants"): - fct.ansible_constants = [] - fct.ansible_constants.append({"description": description, "name": name, "value": value}) - - return fct - - return decorator - - -def AnsibleGatherFacts(fct): - fct.ansible_gather_facts = True - return fct - - -class RunAnsibleRole: - """ - Playbook runner - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - If you're seeing this text, put the --help flag earlier in your list - of command-line arguments, this is a limitation of the CLI parsing library - used by the toolbox. - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - """ - - def __init__( - self, - ansible_vars: dict = None, - role_name: str = None, - group: str = "", - command: str = "", - ): - self.ansible_vars = ansible_vars or {} - self.role_name = role_name - self.group = group - self.command = command - self.clazz = None - - def __str__(self): - return "" - - def _run(self): - if not self.role_name: - raise RuntimeError("Role not set :/") - - if self.ansible_mapped_params: - py_params = self.ansible_vars - self.ansible_vars = { - f"{self.role_name}_{k}": v for k, v in py_params.items() if k != "self" - } - for constant in self.ansible_constants: - self.ansible_vars[f"{self.role_name}_{constant['name']}"] = constant["value"] - - # do not modify the `os.environ` of this Python process - env = os.environ.copy() - - if env.get("ARTIFACT_DIR") is None: - topsail_base_dir = pathlib.Path(env.get("TOPSAIL_BASE_DIR", "/tmp")) - env["ARTIFACT_DIR"] = str(topsail_base_dir / f"topsail_{time.strftime('%Y%m%d')}") - - artifact_dir = pathlib.Path(env["ARTIFACT_DIR"]) - artifact_dir.mkdir(parents=True, exist_ok=True) - - prefix = env.get("ARTIFACT_TOOLBOX_NAME_PREFIX", "") - suffix = env.get("ARTIFACT_TOOLBOX_NAME_SUFFIX", "") - - if env.get("ARTIFACT_EXTRA_LOGS_DIR") is None: - artifact_base_dirname = ( - f"{self.group}__{self.command}" - if self.group and self.command - else "__".join(sys.argv[1:3]) - ) - - previous_extra_count = len(list(artifact_dir.glob("*__*"))) - - name = f"{previous_extra_count:03d}__{prefix}{artifact_base_dirname}{suffix}" - - env["ARTIFACT_EXTRA_LOGS_DIR"] = str(pathlib.Path(env["ARTIFACT_DIR"]) / name) - - artifact_extra_logs_dir = pathlib.Path(env["ARTIFACT_EXTRA_LOGS_DIR"]) - artifact_extra_logs_dir.mkdir(parents=True, exist_ok=True) - - command_name = f"{self.group} {self.command}" - if prefix: - command_name = f"{prefix}/{command_name}" - if suffix: - command_name = f"{command_name}/{suffix}" - - if self.py_command_args: - with open(artifact_extra_logs_dir / "_python.gen.cmd", "w") as f: - print(f"{sys.argv[0]} {self.group} {self.command} \\", file=f) - for key, _value in self.py_command_args.items(): - value = shlex.quote(str(_value)) - print(f" --{key}={value} \\", file=f) - print(" --", file=f) - - with open(artifact_extra_logs_dir / "_python.args.yaml", "w") as f: - print(yaml.dump({self.py_command_name: self.py_command_args}), file=f) - - print(f"Using '{env['ARTIFACT_DIR']}' to store the test artifacts.") - self.ansible_vars["artifact_dir"] = env["ARTIFACT_DIR"] - - print(f"Using '{artifact_extra_logs_dir}' to store extra log files.") - self.ansible_vars["artifact_extra_logs_dir"] = str(artifact_extra_logs_dir) - - if env.get("ANSIBLE_LOG_PATH") is None: - env["ANSIBLE_LOG_PATH"] = str(artifact_extra_logs_dir / "_ansible.log") - print(f"Using '{env['ANSIBLE_LOG_PATH']}' to store ansible logs.") - pathlib.Path(env["ANSIBLE_LOG_PATH"]).parent.mkdir(parents=True, exist_ok=True) - - if env.get("ANSIBLE_CACHE_PLUGIN_CONNECTION") is None: - env["ANSIBLE_CACHE_PLUGIN_CONNECTION"] = str(artifact_dir / "ansible_facts") - print(f"Using '{env['ANSIBLE_CACHE_PLUGIN_CONNECTION']}' to store ansible facts.") - pathlib.Path(env["ANSIBLE_CACHE_PLUGIN_CONNECTION"]).parent.mkdir( - parents=True, exist_ok=True - ) - - # We configure the roles path dynamically appending them to the defaults - topsail_roles_list = [] - - if current_roles_path := env.get("ANSIBLE_ROLES_PATH"): - topsail_roles_list += [current_roles_path] - - topsail_roles_list += [str(entry) for entry in (TOPSAIL_DIR / "projects").glob("*/toolbox")] - - env["ANSIBLE_ROLES_PATH"] = os.pathsep.join(topsail_roles_list) - self.ansible_vars["roles_path"] = env["ANSIBLE_ROLES_PATH"] - - # We configure the collections path dynamically - current_collections_paths = [] - if (collect_path := env.get("ANSIBLE_COLLECTIONS_PATHS")) is not None: - current_collections_paths.append(str(collect_path)) - for path in sys.path: - collections_path = pathlib.Path(path) / "ansible_collections" - if collections_path.exists(): - current_collections_paths.append(str(collections_path)) - env["ANSIBLE_COLLECTIONS_PATHS"] = os.pathsep.join(current_collections_paths) - self.ansible_vars["collections_paths"] = env["ANSIBLE_COLLECTIONS_PATHS"] - - if env.get("ANSIBLE_CONFIG") is None: - env["ANSIBLE_CONFIG"] = str( - TOPSAIL_DIR / "projects/legacy/ansible-config" / "ansible.cfg" - ) - - print(f"Using '{env['ANSIBLE_CONFIG']}' as ansible configuration file.") - - if env.get("ANSIBLE_JSON_TO_LOGFILE") is None: - env["ANSIBLE_JSON_TO_LOGFILE"] = str(artifact_extra_logs_dir / "_ansible.log.json") - print(f"Using '{env['ANSIBLE_JSON_TO_LOGFILE']}' as ansible json log file.") - - # the play file must be in the directory where the 'roles' are - tmp_play_file = tempfile.NamedTemporaryFile( - "w+", - prefix=f"tmp_play_{artifact_extra_logs_dir.name}_", - suffix=".yaml", - dir=os.getcwd(), - delete=False, - ) - - generated_play = [ - { - "name": f"Run {self.role_name} role", - "roles": [self.role_name], - "vars": self.ansible_vars, - } - ] - - remote_hostname = env.get("TOPSAIL_REMOTE_HOSTNAME") - if remote_hostname: - print( - f"Using TOPSAIL_REMOTE_HOSTNAME={remote_hostname}" - ) # value will be censored by OpenShift - if self.ansible_gather_facts: - # gather only env values - generated_play[0]["gather_facts"] = True - generated_play[0]["gather_subset"] = ["env", "!all", "!min"] - else: - generated_play[0]["gather_facts"] = False - - # run remotely - generated_play[0]["hosts"] = "remote" - inventory_fd, path = tempfile.mkstemp() - os.remove( - path - ) # using only the FD. Ensures that the file disappears when this process terminates - inventory_f = os.fdopen(inventory_fd, "w") - - host_properties = [] - if remote_username := env.get("TOPSAIL_REMOTE_USERNAME"): - print( - f"Using TOPSAIL_REMOTE_USERNAME={remote_username}" - ) # value will be censored by OpenShift - - host_properties.append("ansible_user=" + remote_username) - - # Configure OS-specific Ansible variables - if remote_os := env.get("TOPSAIL_REMOTE_OS"): - config = ANSIBLE_OS_CONFIGURATIONS.get(remote_os.lower()) - if not config: - raise ValueError( - f"TOPSAIL Ansible OS configuration not found for TOPSAIL_REMOTE_OS={remote_os}" - ) - host_properties.extend(config) - - inventory_content = f""" -[all:vars] - -[remote] -{remote_hostname} {" ".join(host_properties)} -""" - - print(inventory_content, file=inventory_f) - inventory_f.flush() - else: - # run locally - generated_play[0]["connection"] = "local" - generated_play[0]["hosts"] = "localhost" - generated_play[0]["gather_facts"] = False - - if extra_vars_fname := env.get("TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS"): - try: - with open(extra_vars_fname) as f: - extra_vars_dict = yaml.safe_load(f) - - except yaml.parser.ParserError: - logging.fatal( - f"Could not parse file TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS='{extra_vars_fname}' as yaml ..." - ) - raise - - if not extra_vars_dict: - msg = f"TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS points to an empty file :/ ({extra_vars_dict})" - logging.fatal(msg) - raise ValueError(msg) - - generated_play[0]["vars"] |= extra_vars_dict - - if extra_env_fname := env.get("TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_ENV"): - logging.info( - "Using the extra environment variables found in TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_ENV" - ) - try: - with open(extra_env_fname) as f: - extra_env_dict = yaml.safe_load(f) - - generated_play[0]["environment"] = extra_env_dict - except yaml.parser.ParserError: - logging.fatal( - f"Could not parse file TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_ENV='{extra_env_fname}' as yaml ..." - ) - raise - - generated_play_path = artifact_extra_logs_dir / "_ansible.play.yaml" - with open(generated_play_path, "w") as f: - yaml.dump(generated_play, f) - shutil.copy(generated_play_path, tmp_play_file.name) - - cmd = ["ansible-playbook", "-vv", tmp_play_file.name] - - with open(artifact_extra_logs_dir / "_ansible.env", "w") as f: - for k, v in env.items(): - print(f"{k}={v}", file=f) - - with open(artifact_extra_logs_dir / "_python.cmd", "w") as f: - print(" ".join(map(shlex.quote, sys.argv)), file=f) - - if remote_hostname: - cmd += ["--inventory", f"/proc/{os.getpid()}/fd/{inventory_fd}"] - - sys.stdout.flush() - sys.stderr.flush() - - ret = -1 - try: - run_result = subprocess.run(cmd, env=env, check=False) - ret = run_result.returncode - except KeyboardInterrupt: - print("") - print("Interrupted :/") - sys.exit(1) - finally: - try: - os.remove(tmp_play_file.name) - except FileNotFoundError: - pass # play file was removed, ignore - - if ret != 0: - extra_dir_name = pathlib.Path(env["ARTIFACT_EXTRA_LOGS_DIR"]).name - with open(artifact_extra_logs_dir / "FAILURE.txt", "a") as f: - print(f"[{extra_dir_name}] {' '.join(sys.argv)} --> {ret}", file=f) - - raise SystemExit(ret) diff --git a/projects/legacy/library/common.py b/projects/legacy/library/common.py deleted file mode 100644 index 98bdceccd..000000000 --- a/projects/legacy/library/common.py +++ /dev/null @@ -1,25 +0,0 @@ -import logging - -from . import run - - -def prepare_light_cluster(): - run.run_toolbox("cluster", "wait_fully_awake") - - -def cleanup_cluster(): - busy_cluster_ns_label_key = "busy-cluster.topsail" - busy_cluster_ns_label_value = "yes" - has_busy_cluster_ns = run.run( - f"oc get ns -oname -l{busy_cluster_ns_label_key}={busy_cluster_ns_label_value}", - capture_stdout=True, - ).stdout - if not has_busy_cluster_ns: - logging.info("No busy-cluster namespace detected, nothing to cleanup.") - return - - logging.info( - f"Detected {len(has_busy_cluster_ns.split())} busy-cluster namespaces. Running the cleanup." - ) - - run.run_toolbox("busy_cluster", "cleanup") diff --git a/projects/legacy/library/config.py b/projects/legacy/library/config.py deleted file mode 100644 index 99f7cb4de..000000000 --- a/projects/legacy/library/config.py +++ /dev/null @@ -1,528 +0,0 @@ -import copy -import logging -import os -import pathlib -import re -import shutil -import subprocess -import sys -import threading - -import jsonpath_ng -import yaml - -from . import common, env, run - -logging.getLogger().setLevel(logging.INFO) - -TOPSAIL_DIR = pathlib.Path(common.__file__).parents[3] -VARIABLE_OVERRIDES_FILENAME = "000__ci_metadata/variable_overrides.yaml" -PR_ARG_KEY = "PR_POSITIONAL_ARG_" - -project = None # the project config will be populated in init() - - -class TempValue: - def __init__(self, config, key, value): - self.config = config - self.key = key - self.value = value - self.prev_value = None - - def __enter__(self): - self.prev_value = self.config.get_config(self.key, print=False) - self.config.set_config(self.key, self.value) - - return True - - def __exit__(self, ex_type, ex_value, exc_traceback): - self.config.set_config(self.key, self.prev_value) - - return False # If we returned True here, any exception would be suppressed! - - -class Config: - def __init__(self, testing_dir, config_path): - self.testing_dir = testing_dir - self.config_path = config_path - - if not self.config_path.exists(): - msg = f"Configuration file '{self.config_path}' does not exist :/" - logging.error(msg) - raise ValueError(msg) - - logging.info(f"Loading configuration from {self.config_path} ...") - with open(self.config_path) as config_f: - self.config = yaml.safe_load(config_f) - - def apply_config_overrides( - self, *, ignore_not_found=False, variable_overrides_path=None, log=True - ): - if variable_overrides_path is None: - variable_overrides_path = env.ARTIFACT_DIR / VARIABLE_OVERRIDES_FILENAME - - if not variable_overrides_path.exists(): - logging.debug( - f"apply_config_overrides: {variable_overrides_path} does not exist, nothing to override." - ) - - return - - with open(variable_overrides_path) as f: - variable_overrides = yaml.safe_load(f) - - if not isinstance(variable_overrides, dict): - msg = f"Wrong type for the variable overrides file. Expected a dictionnary, got {variable_overrides.__class__.__name__}" - logging.fatal(msg) - raise ValueError(msg) - - for key, value in variable_overrides.items(): - MAGIC_DEFAULT_VALUE = object() - handled_secretly = True # current_value MUST NOT be printed below. - current_value = self.get_config( - key, - MAGIC_DEFAULT_VALUE, - print=False, - warn=False, - handled_secretly=handled_secretly, - ) - if current_value == MAGIC_DEFAULT_VALUE: - if ignore_not_found: - continue - - if "." in key: - raise ValueError( - f"Config key '{key}' does not exist, and cannot create it at the moment :/" - ) - - self.config[key] = None - - self.set_config(key, value, print=False) - actual_value = self.get_config( - key, print=False - ) # ensure that key has been set, raises an exception otherwise - if log: - logging.info(f"config override: {key} --> {actual_value}") - - def apply_preset(self, name): - try: - values = self.get_config(f'ci_presets["{name}"]', print=False) - except IndexError: - logging.error(f"Preset '{name}' does not exists :/") - raise - - logging.info(f"Applying preset '{name}' ==> {values}") - if values is None: - raise ValueError(f"Preset '{name}' does not exists") - - presets = self.get_config("ci_presets.names", print=False) or [] - if name not in presets: - self.set_config("ci_presets.names", presets + [name]) - - for key, value in values.items(): - if key == "extends": - for extend_name in value: - self.apply_preset(extend_name) - continue - - msg = f"preset[{name}] {key} --> {value}" - logging.info(msg) - with open(env.ARTIFACT_DIR / "presets_applied", "a") as f: - print(msg, file=f) - - self.set_config(key, value, print=False) - - def get_config( - self, jsonpath, default_value=..., warn=True, print=True, handled_secretly=False - ): - try: - value = jsonpath_ng.parse(jsonpath).find(self.config)[0].value - except IndexError as ex: - if default_value != ...: - if warn: - logging.warning( - f"get_config: {jsonpath} --> missing. Returning the default value: {default_value}" - ) - return default_value - - logging.error(f"get_config: {jsonpath} --> {ex}") - raise KeyError(f"Key '{jsonpath}' not found in {self.config_path}") from ex - - if isinstance(value, str) and value.startswith("*$@"): - print = False - - value = self.resolve_reference(value, handled_secretly) - - if print and not handled_secretly: - logging.info(f"get_config: {jsonpath} --> {value}") - - return value - - def set_config(self, jsonpath, value, print=True): - if threading.current_thread().name != "MainThread": - msg = f"set_config({jsonpath}, {value}) cannot be called from a thread, to avoid race conditions." - if os.environ.get("OPENSHIFT_CI") or os.environ.get("PERFLAB_CI"): - logging.error(msg) - with open(env.ARTIFACT_DIR / "SET_CONFIG_CALLED_FROM_THREAD", "a") as f: - print(msg, file=f) - else: - raise RuntimeError(msg) - - try: - self.get_config( - jsonpath, print=False, handled_secretly=True - ) # will raise an exception if the jsonpath does not exist - jsonpath_ng.parse(jsonpath).update(self.config, value) - except Exception as ex: - logging.error(f"set_config: {jsonpath}={value} --> {ex}") - raise - - if print: - logging.info(f"set_config: {jsonpath} --> {value}") - - with open(self.config_path, "w") as f: - yaml.dump(self.config, f, indent=4, default_flow_style=False, sort_keys=False) - - if ( - (shared_dir := os.environ.get("SHARED_DIR")) - and (shared_dir_path := pathlib.Path(shared_dir)) - and shared_dir_path.exists() - ): - with open(shared_dir_path / "config.yaml", "w") as f: - yaml.dump(self.config, f, indent=4, default_flow_style=False, sort_keys=False) - - def save_config_overrides(self): - variable_overrides_path = env.ARTIFACT_DIR / VARIABLE_OVERRIDES_FILENAME - - if not variable_overrides_path.exists(): - logging.debug( - f"save_config_overrides: {variable_overrides_path} does not exist, nothing to save." - ) - self.config["overrides"] = {} - return - - with open(variable_overrides_path) as f: - variable_overrides = yaml.safe_load(f) - - self.config["overrides"] = variable_overrides - - def apply_preset_from_pr_args(self): - for config_key in self.get_config("$", print=False).keys(): - if not config_key.startswith(PR_ARG_KEY): - continue - if config_key == f"{PR_ARG_KEY}0": - continue - - presets = self.get_config(config_key) - if not presets: - continue - for preset in presets.strip().split(" "): - self.apply_preset(preset) - - def detect_apply_light_profile(self, profile, name_suffix="light"): - if os.environ.get("OPENSHIFT_CI"): - job_name_safe = os.environ.get("JOB_NAME_SAFE", ...) - - if job_name_safe is ...: - raise RuntimeError("Running in OpenShift CI but JOB_NAME_SAFE not set :/") - - if job_name_safe != name_suffix and not job_name_safe.endswith(f"-{name_suffix}"): - return False - - logging.info( - f"Running a '{name_suffix}' test ({job_name_safe}), applying the '{profile}' profile" - ) - - self.apply_preset(profile) - - if os.environ.get("HOSTNAME") == "light-prepare": - common.prepare_light_cluster() - - return True - - logging.info("Not running in OpenShift CI, no light environment to detect.") - - return False - - def detect_apply_metal_profile(self, profile): - platform_type_cmd = run.run( - "oc get infrastructure/cluster -ojsonpath={.status.platformStatus.type}", - capture_stdout=True, - capture_stderr=True, - check=False, - ) - if platform_type_cmd.returncode != 0: - logging.warning(f"Failed to get the platform type: {platform_type_cmd.stderr.strip()}") - logging.warning("Ignoring the metal profile check.") - return False - - platform_type = platform_type_cmd.stdout - logging.info( - f"detect_apply_metal_profile: infrastructure/cluster.status.platformStatus.type = {platform_type}" - ) - if platform_type not in ("BareMetal", "None", "IBMCloud"): - logging.info( - "detect_apply_metal_profile: Assuming not running in a bare-metal environment." - ) - return False - logging.info( - f"detect_apply_metal_profile: Assuming running in a bare-metal environment. Applying the '{profile}' profile." - ) - - self.apply_preset(profile) - return True - - def detect_apply_cluster_profile(self, node_profiles): - cluster_nodes_cmd = run.run( - "oc get nodes -oname", capture_stdout=True, capture_stderr=True, check=False - ) - if cluster_nodes_cmd.returncode != 0: - logging.warning(f"Failed to get the cluster nodes: {cluster_nodes_cmd.stderr.strip()}") - logging.warning("Ignoring the cluster profile check.") - return False - - for node_name in cluster_nodes_cmd.stdout.split(): - for node, profile in node_profiles.items(): - if f"node/{node}" != node_name: - continue - - self.apply_preset(profile) - return profile - - return False - - def resolve_reference(self, value, handled_secretly=False): - if not isinstance(value, str): - return value - if "@" not in value: - return value - - # --- # - - def secret_file_dereference(): - if not handled_secretly: - msg = f"{value} is a secret dereference, but get_config(..., handled_secretly=False). Aborting" - logging.fatal(msg) - raise ValueError(msg) - - ref_key = value.removeprefix("*$@") - ref_value = self.get_config(ref_key, print=False) - - secret_dir = pathlib.Path( - os.environ[self.get_config("secrets.dir.env_key", print=False)] - ) - secret_value = (secret_dir / ref_value).read_text().strip() - - return secret_value - - # --- # - - def simple_dereference(): - ref_key = value[1:] - return self.get_config(ref_key) - - def multi_dereference(): - new_value = value - for ref in re.findall(r"\{@.*?\}", value): - ref_key = ref.strip("{@}") - ref_value = self.get_config(ref_key, print=False) - new_value = new_value.replace(ref, str(ref_value)) - - return new_value - - # --- # - - if value.startswith("*$@"): - return secret_file_dereference() - - if value.startswith("*@"): - # value can be printed here, it's a reference to a secret, not a secret value - msg = f"resolve_reference: '*@' references not supported (not sure how to handle it wrt to secrets) --> {value}" - logging.fatal(msg) - raise ValueError(msg) - - if not (value.startswith("@") or "{@" in value): - # don't go further if the derefence anchor isn't found - return value - - # --- # - - new_value = simple_dereference() if value.startswith("@") else multi_dereference() - - if not handled_secretly: - logging.info(f"resolve_reference: {value} ==> '{new_value}'") - - return copy.deepcopy(new_value) - - -def _set_config_environ(testing_dir): - config_path_final = pathlib.Path(env.ARTIFACT_DIR / "config.yaml") - - config_file_src = None - if env_config_file_src := os.environ.get("TOPSAIL_FROM_CONFIG_FILE"): - config_file_src = env_config_file_src - logging.info( - f"Loading the configuration from TOPSAIL_FROM_CONFIG_FILE={config_file_src} ..." - ) - - elif ( - shared_dir_file_src := pathlib.Path(os.environ.get("SHARED_DIR", "/not-a-directory")) - / "config.yaml" - ) and shared_dir_file_src.exists(): - config_file_src = shared_dir_file_src - logging.info(f"Reloading the config file from SHARED_DIR {config_file_src} ...") - else: - config_file_src = testing_dir / "config.yaml" - logging.info( - f"Reloading the config file from TOPSAIL project directory {config_file_src} ..." - ) - - os.environ["TOPSAIL_FROM_CONFIG_FILE"] = str(config_path_final) - - if "TOPSAIL_FROM_COMMAND_ARGS_FILE" not in os.environ: - os.environ["TOPSAIL_FROM_COMMAND_ARGS_FILE"] = str(testing_dir / "command_args.yml.j2") - - if not pathlib.Path(config_file_src) == config_path_final: - logging.info(f"Copying the configuration from {config_file_src} to the artifact dir ...") - shutil.copyfile(config_file_src, config_path_final) - - return config_path_final - - -def get_command_arg(group, command, arg, prefix=None, suffix=None, mute=False): - try: - if not mute: - logging.info(f"get_command_arg: {group} {command} {arg}") - - proc = run.run_toolbox_from_config( - group, - command, - show_args=arg, - prefix=prefix, - suffix=suffix, - check=True, - run_kwargs={ - "capture_stdout": True, - "capture_stderr": True, - "log_command": (not mute), - }, - ) - except subprocess.CalledProcessError as e: - logging.error(e.stderr.strip().decode("ascii", "ignore")) - raise - - return proc.stdout.strip() - - -def set_jsonpath(config, jsonpath, value): - get_jsonpath(config, jsonpath) # will raise an exception if the jsonpath does not exist - jsonpath_ng.parse(jsonpath).update(config, value) - - -def get_jsonpath(config, jsonpath): - return jsonpath_ng.parse(jsonpath).find(config)[0].value - - -def test_skip_list(): - if len(sys.argv) < 2: - logging.info( - "test_skip_list: cannot determinate the current subcommand. Not processing the skip list." - ) - return - - current_subcommand = sys.argv[1] - logging.info(f"Currently running the subcommand '{current_subcommand}'") - - exec_list = project.get_config("exec_list", None, print=False) - if exec_list is None: - logging.warning("The exec_list isn't defined in this project.") - exec_list = {} - - NOT_FOUND = object() - exec_this_subcommand = exec_list.get(current_subcommand, NOT_FOUND) - if exec_this_subcommand is NOT_FOUND: - logging.info( - f"Subcommand '{current_subcommand}' is not defined in the exec list. Executing this command by default." - ) - return - - if exec_this_subcommand is False: - logging.fatal( - f"Subcommand '{current_subcommand}' is disabled in the exec list. Stopping happily this execution." - ) - with open(env.ARTIFACT_DIR / "SKIPPED", "w") as f: - print("Skipped because part of the \\skip list", file=f) - raise SystemExit(0) - - if exec_this_subcommand is not True and exec_list.get("_only_", False): - logging.fatal( - f"Only flag is set, and subcommand '{current_subcommand}' is not enabled in the exec list. Stopping happily this execution." - ) - with open(env.ARTIFACT_DIR / "SKIPPED", "w") as f: - print("Skipped because not part of the \\only list", file=f) - - raise SystemExit(0) - - # not in the skip list - # not the only command to execute - # continue happilly =:-) - - pass - - -def init(testing_dir, apply_preset_from_pr_args=False, apply_config_overrides=True): - global project - - if project: - logging.info("config.init: project config already configured.") - return - - config_path = _set_config_environ(testing_dir) - project = Config(testing_dir, config_path) - - if os.environ.get("TOPSAIL_LOCAL_CI_MULTI") == "true": - logging.info( - "config.init: running in a local-ci multi Pod, skipping apply_config_overrides and apply_preset_from_pr_args." - ) - return - - if not apply_config_overrides: - logging.info( - "config.init: running with 'apply_config_overrides', skipping the overrides. Saving it as 'overrides' field in the project configuration." - ) - project.save_config_overrides() - return - - repo_var_overrides = TOPSAIL_DIR / VARIABLE_OVERRIDES_FILENAME - - if repo_var_overrides.exists(): - logging.info(f"Found '{repo_var_overrides}', apply the variables overrides from it.") - project.apply_config_overrides(variable_overrides_path=repo_var_overrides) - - ci_presets_to_apply = project.get_config("ci_presets.to_apply", [], warn=False) - if isinstance(ci_presets_to_apply, str): - ci_presets_to_apply = [ci_presets_to_apply] - - for preset in ci_presets_to_apply: - project.apply_preset(preset) - - variable_overrides_to_apply = project.get_config( - "ci_presets.variable_overrides", {}, warn=False - ) - for var_name, var_value in variable_overrides_to_apply.items(): - project.set_config(var_name, var_value) - - if repo_var_overrides.exists(): - # reapply to force overrides on top of presets - project.apply_config_overrides(variable_overrides_path=repo_var_overrides, log=False) - - project.apply_config_overrides() - - if apply_preset_from_pr_args: - project.apply_preset_from_pr_args() - # reapply to force overrides on top of presets - project.apply_config_overrides(log=False) - - test_skip_list() - - pass diff --git a/projects/legacy/library/env.py b/projects/legacy/library/env.py deleted file mode 100644 index 1c58f0dca..000000000 --- a/projects/legacy/library/env.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -import os -import pathlib -import threading -import time -import traceback - -### -# The code below required to properly set the ARTIFACT_DIR in the -# thead local storage, so that threads don't share the same -# ARTIFACT_DIR value (and don't update the shared value) -### - - -class MyThread(threading.Thread): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.parent_artifact_dict = None - - def start(self): - self.parent_artifact_dict = get_tls_artifact_dir() - super().start() - - def run(self): - _set_tls_artifact_dir(self.parent_artifact_dict) - super().run() - - -threading.Thread = MyThread - - -def __getattr__(name): - if name == "ARTIFACT_DIR": - return get_tls_artifact_dir() - - return globals()[name] - - -_main_artifact_dir = None -_tls_artifact_dir = threading.local() - - -def get_tls_artifact_dir(): - return _tls_artifact_dir.val - - -def _set_tls_artifact_dir(value): - _tls_artifact_dir.val = value - - -### -# end of the thread local storage code -### - - -def init(): - if "ARTIFACT_DIR" in os.environ: - artifact_dir = pathlib.Path(os.environ["ARTIFACT_DIR"]) - - else: - env_topsail_base_dir = pathlib.Path(os.environ.get("TOPSAIL_BASE_DIR", "/tmp")) - artifact_dir = env_topsail_base_dir / f"forge_{time.strftime('%Y%m%d-%H%M')}" - - artifact_dir.mkdir(parents=True, exist_ok=True) - os.environ["ARTIFACT_DIR"] = str(artifact_dir) - - artifact_dir.mkdir(parents=True, exist_ok=True) - _set_tls_artifact_dir(artifact_dir) - - -def NextArtifactDir(name, *, lock=None, counter_p=None): - if lock: - with lock: - next_count = counter_p[0] - counter_p[0] += 1 - else: - next_count = next_artifact_index() - - dirname = get_tls_artifact_dir() / f"{next_count:03d}__{name}" - - return TempArtifactDir(dirname) - - -class TempArtifactDir: - def __init__(self, dirname): - self.dirname = pathlib.Path(dirname) - self.previous_dirname = None - - def __enter__(self): - self.previous_dirname = get_tls_artifact_dir() - os.environ["ARTIFACT_DIR"] = str(self.dirname) - self.dirname.mkdir(exist_ok=True) - - _set_tls_artifact_dir(self.dirname) - - return True - - def __exit__(self, ex_type, ex_value, exc_traceback): - global _ARTIFACT_DIR - - if ex_value: - logging.error(f"Caught exception {ex_type.__name__}: {ex_value}") - with open(get_tls_artifact_dir() / "FAILURE.txt", "a") as f: - print(f"{ex_type.__name__}: {ex_value}", file=f) - print( - "".join(traceback.format_exception(None, value=ex_value, tb=exc_traceback)), - file=f, - ) - - os.environ["ARTIFACT_DIR"] = str(self.previous_dirname) - _set_tls_artifact_dir(self.previous_dirname) - - return False # If we returned True here, any exception would be suppressed! - - -def next_artifact_index(): - return len(list(get_tls_artifact_dir().glob("*__*"))) diff --git a/projects/legacy/library/export.py b/projects/legacy/library/export.py deleted file mode 100755 index b30e3e36b..000000000 --- a/projects/legacy/library/export.py +++ /dev/null @@ -1,118 +0,0 @@ -#! /usr/bin/env python - -import functools -import json -import logging -import os -import pathlib -import subprocess -import sys - -import fire - -from projects.legacy.library import config, env, run - -logging.getLogger().setLevel(logging.INFO) - - -def init(): - env.init() - config_file = os.environ.get("TOPSAIL_FROM_CONFIG_FILE") - if not config_file: - raise RuntimeError( - "TOPSAIL_FROM_CONFIG_FILE must be set. Please source your `configure.sh` before running this file." - ) - - config.init(pathlib.Path(config_file).parent) - - -def entrypoint(): - def decorator(fct): - @functools.wraps(fct) - def wrapper(*args, **kwargs): - init() - fct(*args, **kwargs) - - return wrapper - - return decorator - - -@entrypoint() -def export_artifacts(artifacts_dirname, test_step=None): - if test_step and os.environ.get("PERFLAB_CI") == "true": - logging.info("export_artifacts: in the PerfLab. Ignoring step export.") - return - - if not config.project.get_config("export_artifacts.enabled"): - logging.info("export_artifacts.enabled not set, nothing to do here.") - return - - bucket = config.project.get_config("export_artifacts.bucket") - path_prefix = config.project.get_config("export_artifacts.path_prefix") - if not path_prefix: - path_prefix = "" - - if os.environ.get("OPENSHIFT_CI") == "true": - job_spec = json.loads(os.environ["JOB_SPEC"]) - pull_number = job_spec["refs"]["pulls"][0]["number"] - job = job_spec["job"] - build_id = job_spec["buildid"] - - run_id = f"prow/{pull_number}/{build_id}" - - elif os.environ.get("PERFLAB_CI") == "true": - logging.warning("No way to get the run identifiers from Jenkins in the PERFLAB_CI") - - build_number = os.environ["JENKINS_BUILD_NUMBER"] - job = os.environ["JENKINS_JOB"] # "job/ExternalTeams/job/RHODS/job/topsail" - job_id = job[4:].replace("/job/", "_") - - run_id = f"middleware_jenkins/{job_id}/{build_number}" - else: - logging.error("CI engine not recognized, cannot build the run id ...") - raise ValueError("CI engine not recognized, cannot build the run id ...") - - if test_step: - run_id += f"/{test_step}" - - export_dest = f"s3://{bucket}/{path_prefix}/{run_id}" - export_url = f"https://{bucket}.s3.amazonaws.com/index.html#{path_prefix}/{run_id}/" - with open(env.ARTIFACT_DIR / "export.url", "w") as f: - print(export_url, file=f) - - config.project.set_config("export_artifacts.dest", export_url) - - logging.info(f"Exporting to {export_dest} ({export_url})") - aws_creds_filename = config.project.get_config("secrets.aws_credentials") - run.run( - f'AWS_SHARED_CREDENTIALS_FILE="$PSAP_ODS_SECRET_PATH/{aws_creds_filename}" aws s3 cp --recursive "{artifacts_dirname}" "{export_dest}" --acl public-read &> {env.ARTIFACT_DIR / "aws_s3_cp.log"}' - ) - - -class Export: - """ - Commands for launching the artifacts export - """ - - def __init__(self): - self.export_artifacts = export_artifacts - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - fire.Fire(Export()) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - logging.error(f"Command '{e.cmd}' failed --> {e.returncode}") - sys.exit(1) - except KeyboardInterrupt: - print() # empty line after ^C - logging.error("Interrupted.") - sys.exit(1) diff --git a/projects/legacy/library/run.py b/projects/legacy/library/run.py deleted file mode 100644 index f14dfe9c9..000000000 --- a/projects/legacy/library/run.py +++ /dev/null @@ -1,309 +0,0 @@ -import itertools -import json -import logging -import os -import pathlib -import signal -import subprocess -import traceback - -try: - import joblib -except ImportError: - logging.warning( - "Joblib package not available. Won't run jobs with multiple Parallel processes." - ) - joblib = None - -from . import config, env - -logging.getLogger().setLevel(logging.INFO) - -FORGE_HOME = pathlib.Path(__file__).resolve().parent.parent.parent.parent - -# create new process group, become its leader, except if we're already pid 1 (defacto group leader, setpgrp gets permission denied error) -try: - if os.getpid() != 1: - os.setpgrp() -except Exception as e: - logging.warning(f"Cannot call os.setpgrp: {e}") - - -class SignalInterrupt(SystemExit): - def __init__(self, sig, frame): - self.sig = sig - self.frame = frame - - def __str__(self): - return f"SignalInterrupt(sig={self.sig})" - - -def raise_signal(sig, frame): - raise SignalInterrupt(sig, frame) - - -signal.signal(signal.SIGINT, raise_signal) -signal.signal(signal.SIGTERM, raise_signal) - - -def run_toolbox_from_config( - group, - command, - prefix=None, - suffix=None, - show_args=None, - extra=None, - artifact_dir_suffix=None, - mute_stdout=False, - check=True, - run_kwargs=None, -): - if extra is None: - extra = {} - - if run_kwargs is None: - run_kwargs = {} - - run_kwargs["cwd"] = str(FORGE_HOME) - - kwargs = {} - if prefix is not None: - kwargs["prefix"] = prefix - if suffix is not None: - kwargs["suffix"] = suffix - if extra is not None: - kwargs["extra"] = extra - if show_args: - kwargs["show_args"] = show_args - - if mute_stdout: - run_kwargs["capture_stdout"] = True - - if check is not None: - run_kwargs["check"] = check - - env_vals = [f'ARTIFACT_DIR="{env.ARTIFACT_DIR}"'] - if artifact_dir_suffix is not None: - env_vals.append(f'ARTIFACT_TOOLBOX_NAME_SUFFIX="{artifact_dir_suffix}"') - - cmd_env = " ".join(env_vals) - - return run( - f"{cmd_env} bin/run_ansible_toolbox.py from_config {group} {command} {_dict_to_run_toolbox_args(kwargs)}", - **run_kwargs, - ) - - -def _dict_to_run_toolbox_args(args_dict): - args = [] - for k, v in args_dict.items(): - if isinstance(v, dict) or isinstance(v, list): - json.dumps(v) - arg = f'--{k}="{v}"' - else: - str(v).replace("'", "'") - arg = f"--{k}='{v}'" - args.append(arg) - - return " ".join(args) - - -def run_toolbox( - group, - command, - artifact_dir_suffix=None, - run_kwargs=None, - mute_stdout=None, - mute_stderr=None, - check=None, - **kwargs, -): - if run_kwargs is None: - run_kwargs = {} - - if mute_stdout: - run_kwargs["capture_stdout"] = True - - if mute_stderr: - run_kwargs["capture_stderr"] = True - - if check is not None: - run_kwargs["check"] = check - - env_vals = [f'ARTIFACT_DIR="{env.ARTIFACT_DIR}"'] - if artifact_dir_suffix is not None: - env_vals.append(f'ARTIFACT_TOOLBOX_NAME_SUFFIX="{artifact_dir_suffix}"') - - cmd_env = " ".join(env_vals) - - run_kwargs["cwd"] = str(FORGE_HOME) - - return run( - f"{cmd_env} bin/run_toolbox.py {group} {command} {_dict_to_run_toolbox_args(kwargs)}", - **run_kwargs, - ) - - -def run( - command, - capture_stdout=False, - capture_stderr=False, - check=True, - protect_shell=True, - cwd=None, - stdin_file=None, - log_command=True, - decode_stdout=True, - decode_stderr=True, -): - if log_command: - logging.info(f"run: {command}") - - args = {} - - args["cwd"] = cwd - args["shell"] = True - - if capture_stdout: - args["stdout"] = subprocess.PIPE - if capture_stderr: - args["stderr"] = subprocess.PIPE - if check: - args["check"] = True - if stdin_file: - if not hasattr(stdin_file, "fileno"): - raise ValueError("Argument 'stdin_file' must be an open file (with a file descriptor)") - args["stdin"] = stdin_file - - if protect_shell: - command = f"set -o errexit;set -o pipefail;set -o nounset;set -o errtrace;{command}" - - proc = subprocess.run(command, **args) - - if capture_stdout and decode_stdout: - proc.stdout = proc.stdout.decode("utf8") - if capture_stderr and decode_stderr: - proc.stderr = proc.stderr.decode("utf8") - - return proc - - -class Parallel: - def __init__(self, name, exit_on_exception=True, dedicated_dir=True): - self.name = name - self.parallel_tasks = None - self.exit_on_exception = exit_on_exception - self.dedicated_dir = dedicated_dir - - def __enter__(self): - self.parallel_tasks = [] - - return self - - def delayed(self, function, *args, **kwargs): - self.parallel_tasks += ( - [joblib.delayed(function)(*args, **kwargs)] if joblib else [(function, args, kwargs)] - ) - - def __exit__(self, ex_type, ex_value, exc_traceback): - if ex_value: - logging.warning( - f"An exception occured while preparing the '{self.name}' Parallel execution ..." - ) - return False - - if self.dedicated_dir: - context = env.NextArtifactDir(self.name) - else: - context = open("/dev/null") # dummy context - - with context: - try: - if joblib: - joblib.Parallel(n_jobs=-1, backend="threading")(self.parallel_tasks) - else: - logging.info("joblib not available, running the delayed function sequentially.") - - for function, args, kwargs in self.parallel_tasks: - function(*args, **kwargs) - except Exception as e: - if not self.exit_on_exception: - raise e - - traceback.print_exc() - - logging.error( - f"Exception caught during the '{self.name}' Parallel execution. Exiting." - ) - # kill all processes in my group - # (the group was started with the os.setpgrp() above) - os.killpg(0, signal.SIGKILL) - print("bye") - raise SystemExit(1) from None - - return False # If we returned True here, any exception would be suppressed! - - -def run_and_catch(exc, fct, *args, **kwargs): - """ - Helper function for chaining multiple functions without swallowing exceptions - Example: - - exc = None - exc = run.run_and_catch( - exc, - run.run_toolbox, "kserve", "capture_operators_state", run_kwargs=dict(capture_stdout=True), - ) - - exc = run.run_and_catch( - exc, - run.run_toolbox, "cluster", "capture_environment", run_kwargs=dict(capture_stdout=True), - ) - - if exc: raise exc - """ - if not (exc is None or isinstance(exc, Exception)): - raise ValueError(f"exc={exc} should be None or an Exception ({exc.__class__})") - - try: - fct(*args, **kwargs) - except Exception as e: - logging.error(f"{e.__class__.__name__}: {e}") - exc = exc or e - return exc - - -# applies the different values of the fields and runs fct(*args, **kwargs) -# example: iterable_fields=[my.config.value1, my.config.value2] and: -# my.config.value1: [1, 2] -# my.config.value2: [a, b] -# this will run 4 times, with -# - my.config.value1: 1, my.config.value2: a -# - my.config.value1: 1, my.config.value2: b -# - my.config.value1: 2, my.config.value2: a -# - my.config.value1: 2, my.config.value2: b -def run_iterable_fields(iterable_fields, fct, *args, **kwargs): - if not iterable_fields: - # nothing to do - fct(*args, **kwargs) - return - - iterable_kv = {} - for iterable_key in iterable_fields: - iterable_values = config.project.get_config(iterable_key, print=False) - if not isinstance(iterable_values, list): - continue - iterable_kv[iterable_key] = iterable_values - - kv_list = [[(key, v) for v in iterable_kv[key]] for key in iterable_kv] - for kv_entry in itertools.product(*kv_list): - for k, v in kv_entry: - config.project.set_config(k, v) - - fct(*args, **kwargs) - - for iter_key, iter_values in iterable_kv.items(): - config.project.set_config(iter_key, iter_values, print=False) - - for iter_key, iter_values in iterable_kv.items(): - config.project.set_config(iter_key, iter_values, print=False) diff --git a/projects/legacy/library/run_toolbox.py b/projects/legacy/library/run_toolbox.py deleted file mode 100755 index 9e43ce4a1..000000000 --- a/projects/legacy/library/run_toolbox.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -import sys - -try: - import fire -except ModuleNotFoundError: - print( - "The toolbox requires the Python `fire` package, see requirements.txt for a full list of requirements" - ) - sys.exit(1) - -import projects.legacy.library.ansible_toolbox - - -def main(no_exit=False): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - # Launch CLI, get a runnable - runnable = None - try: - runnable = fire.Fire(projects.legacy.library.ansible_toolbox.Toolbox()) - except fire.core.FireExit: - if not no_exit: - raise - - # Run the actual workload - try: - if hasattr(runnable, "_run"): - runnable._run() - else: - # CLI didn't resolve completely - either by lack of arguments - # or use of `--help`. This is okay. - pass - except SystemExit as e: - if not no_exit: - sys.exit(e.code) - - -if __name__ == "__main__": - main() diff --git a/projects/legacy/library/sizing.machines b/projects/legacy/library/sizing.machines deleted file mode 100644 index fbdf6190b..000000000 --- a/projects/legacy/library/sizing.machines +++ /dev/null @@ -1,69 +0,0 @@ -# General purpose - -m5.xlarge, 4 vCPU, 16 GiB RAM, $0.096 -m5.2xlarge, 8 vCPU, 32 GiB RAM, $0.384 -m5.4xlarge, 16 vCPU, 64 GiB RAM, $0.768 -m5.8xlarge, 32 vCPU, 128 GiB RAM, $1.536 -m5.12xlarge, 48 vCPU, 192 GiB RAM, $2.304 -m5.16xlarge, 64 vCPU, 256 GiB RAM, $3.072 -m5.24xlarge, 96 vCPU, 384 GiB RAM, $4.608 - -# Memory optimized -r5.xlarge, 4 vCPU, 32 GiB RAM, $0.252 -r6i.xlarge, 4 vCPU, 32 GiB RAM, $0.252 -z1d.xlarge, 4 vCPU, 32 GiB RAM, $0.372 -r5.2xlarge, 8 vCPU, 64 GiB RAM, $0.504 -r6i.2xlarge, 8 vCPU, 64 GiB RAM, $0.504 -z1d.2xlarge, 8 vCPU, 64 GiB RAM, $0.744 -z1d.3xlarge, 12 vCPU, 96 GiB RAM, $1.116 -r5.4xlarge, 16 vCPU, 128 GiB RAM, $1.008 -r6i.4xlarge, 16 vCPU, 128 GiB RAM, $1.008 -z1d.6xlarge, 24 vCPU, 192 GiB RAM, $2.232 -r5.8xlarge, 32 vCPU, 256 GiB RAM, $2.016 -r6i.8xlarge, 32 vCPU, 256 GiB RAM, $2.016 -r5.12xlarge, 48 vCPU, 384 GiB RAM, $3.024 -r6i.12xlarge, 48 vCPU, 384 GiB RAM, $3.024 -z1d.12xlarge, 48 vCPU, 384 GiB RAM, $4.464 -r5.16xlarge, 64 vCPU, 512 GiB RAM, $4.032 -r6i.16xlarge, 64 vCPU, 512 GiB RAM, $4.032 -r5.24xlarge, 96 vCPU, 768 GiB RAM, $6.048 -r6i.24xlarge, 96 vCPU, 768 GiB RAM, $6.048 -r6i.32xlarge, 128 vCPU, 1000 GiB RAM, $8.064 - -# Compute optimized -# c5.xlarge, 4 vCPU, 8 GiB RAM, $0.17 -c5.2xlarge, 8 vCPU, 16 GiB RAM, $0.34 -c5.4xlarge, 16 vCPU, 32 GiB RAM, $0.68 -c5.9xlarge, 36 vCPU, 72 GiB RAM, $1.53 -c5.12xlarge, 48 vCPU, 96 GiB RAM, $2.04 -c5.18xlarge, 72 vCPU, 144 GiB RAM, $3.06 -c5.24xlarge, 96 vCPU, 192 GiB RAM, $4.08 - -# Accelated computing -#g4dn.xlarge, 4 vCPU, 16 GiB RAM, $0.526, 1 GPU, 16 GiB GPU RAM -g4dn.2xlarge, 8 vCPU, 32 GiB RAM, $0.752, 1 GPU, 16 GiB GPU RAM -g4dn.4xlarge, 16 vCPU, 64 GiB RAM, $1.204, 1 GPU, 16 GiB GPU RAM -g4dn.12xlarge, 48 vCPU, 192 GiB RAM, $3.912, 4 GPU, 64 GiB GPU RAM -g5.xlarge, 4 vCPU, 16 GiB RAM, $1.006, 1 GPU, 24 GiB GPU RAM -g5.2xlarge, 8 vCPU, 32 GiB RAM, $1.212, 1 GPU, 24 GiB GPU RAM -g5.4xlarge, 16 vCPU, 64 GiB RAM, $1.624, 1 GPU, 24 GiB GPU RAM -g5.8xlarge, 32 vCPU, 128 GiB RAM, $2.448, 1 GPU, 24 GiB GPU RAM -g5.12xlarge, 48 vCPU, 192 GiB RAM, $5.672, 1 GPU, 96 GiB GPU RAM -#p3.2xlarge, 8 vCPU, 61 GiB RAM, $3.06, 1 GPU -#p3.8xlarge, 32 vCPU, 244 GiB RAM, $12.24, 4 GPU -#p3.16xlarge, 64 vCPU, 488 GiB RAM, $24.48, 8 GPU - -# Not an OSD machine -m6a.xlarge, 4 vcpu, 16 Gib RAM, $0.173 -m6a.2xlarge, 8 vcpu, 32 Gib RAM, $0.345 -m6i.2xlarge, 8 vcpu, 32 Gib RAM, $0.384 -m5a.2xlarge, 8 vcpu, 32 Gib RAM, $0.344 -m6i.4xlarge, 16 vcpu, 64 Gib RAM, $0.0 -m6i.8xlarge, 32 vcpu, 128 Gib RAM, $0.0 -m6i.12xlarge, 48 vcpu, 192 Gib RAM, $2.304 -m6i.16xlarge, 64 vCPU, 256 GiB RAM, $3.072 - -# Bare-metal scale lab machines -Dell R650, 112 vcpu, 512 Gib RAM, $0 -Dell R640, 80 vcpu, 384 Gib RAM, $0 -Dell FC640, 64 vpcu, 384 Gib RAM, $0 diff --git a/projects/legacy/library/sizing.py b/projects/legacy/library/sizing.py deleted file mode 100644 index 19db43aa6..000000000 --- a/projects/legacy/library/sizing.py +++ /dev/null @@ -1,144 +0,0 @@ -#! /usr/bin/env python - -import logging -import math -import os -import pathlib -import types - -logging.getLogger().setLevel(logging.INFO) - -MACHINES_FILE = pathlib.Path(os.path.dirname(os.path.realpath(__file__))) / "sizing.machines" - - -def parse_machines(): - machines = {} - with open(MACHINES_FILE) as f: - for _line in f.readlines(): - line = _line.strip() - if line.startswith("# "): - group = line.strip("# ") - - if not line or line.startswith("#"): - continue - - instance, cpu, memory, price, *accel = line.split(", ") - - entry = types.SimpleNamespace() - entry.cpu = int(cpu.split()[0]) - entry.memory = int(memory.split()[0]) - entry.price = float(price[1:]) - entry.group = group - entry.name = instance - - machines[entry.name] = entry - return machines - - -RESERVED_CPU = 2 -RESERVED_MEM = 4 - -EXTRA_USERS = 1 # count as if there was +10% of users - -MAX_POD_PER_NODE = 250 - 15 # 250 pods allocatable, 15 pods - - -def main(machine_type, user_count, cpu, memory): - machines = parse_machines() - - pod_size = {"cpu": cpu, "memory": memory} - - machine_size = machines[machine_type] - - logging.info(f"Reserved cpu={RESERVED_CPU}, mem={RESERVED_MEM}") - logging.info(f"Machine type: {machine_type} --> {machine_size}") - logging.info(f"Pod size: cpu={cpu}, mem={memory}Gi") - logging.info("") - - total_cpu_count = pod_size["cpu"] * user_count - total_memory_count = pod_size["memory"] * user_count - - machine_count_cpu = total_cpu_count / (machine_size.cpu - RESERVED_CPU) - machine_count_memory = total_memory_count / (machine_size.memory - RESERVED_MEM) - - logging.info(f"Memory requirement: {machine_count_memory:.1f} x {machine_type}") - logging.info(f"CPU requirement: {machine_count_cpu:.1f} x {machine_type}") - logging.info("") - - machine_exact_count = max([machine_count_cpu, machine_count_memory]) - machine_count = math.ceil(machine_exact_count) - - pods_per_machine = math.ceil(user_count / machine_count) - - # ensure that the expected pod/machine - if pods_per_machine > MAX_POD_PER_NODE: - logging.info( - f"Computation gives {pods_per_machine} Pods per node on {machine_count}. " - f"Increasing the node count to stay below {MAX_POD_PER_NODE} pods/node." - ) - pods_per_machine = MAX_POD_PER_NODE - machine_count = math.ceil(user_count / pods_per_machine) - - logging.info( - f"Provisioning {machine_count} {machine_type} machines " - f"for running {user_count} users with the pod size cpu={cpu}, mem={memory}" - ) - unallocated_cpu = machine_size.cpu - pod_size["cpu"] * pods_per_machine - unallocated_mem = machine_size.memory - pod_size["memory"] * pods_per_machine - - logging.info( - f"Expecting {pods_per_machine:d} pods per node " - f"({unallocated_cpu:.3f} cpu and {unallocated_mem:.2f}Gi of memory " - f"not allocated per node)" - ) - - AWS_MAX_VOLUMES_PER_NODE = 26 - if "xlarge" in machine_type and pods_per_machine > AWS_MAX_VOLUMES_PER_NODE: - logging.info( - f"WARNING: if the Pods have AWS volumes, " - f"this configuration won't work (only {AWS_MAX_VOLUMES_PER_NODE} volumes " - f"per node is working)" - ) - logging.info( - "See https://docs.openshift.com/container-platform/4.12/storage/persistent_storage/persistent-storage-aws.html" - ) - logging.info("See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html") - - return machine_count - - -if __name__ == "__main__": - import sys - - try: - machine_type, _user_count, _cpu, _memory = sys.argv[1:] - - user_count = int(_user_count) - cpu = float(_cpu) - memory = float(_memory) - except ValueError: - logging.error( - f"expected 4 arguments: `MACHINE_TYPE, USER_COUNT, CPU, MEMORY`, " - f"got {len(sys.argv[1:])}: {sys.argv[1:]}" - ) - logging.info(f"""Example: -MACHINE="Dell FC640" -USERS=1000 - -CPU=0.2 -MEM=0.750 -# or -CPU=1 -MEM=4 - -{sys.argv[0]} "$MACHINE" "$USERS" "$CPU" "$MEM" -""") - sys.exit(1) - - try: - sys.exit( - main(machine_type, user_count, cpu, memory) - ) # returns the number of nodes required - except Exception as e: - logging.error(f"'{' '.join(sys.argv)}' failed: {e.__class__.__name__}: {e}") - sys.exit(0) # 0 means that an error occured From 38ab485bf2757e72df24ce220c3ce2285969a874 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:18:21 +0200 Subject: [PATCH 0878/1072] pyproject.toml: remove matrixbenchmarking and the legacy projects --- pyproject.toml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8a3f8d4c2..18b65ee01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,20 +68,6 @@ caliper = [ prometheus_metrics = [ "pyyaml>=6.0", ] -matrix_benchmarking = [ - "pandas", - "dash", - "plotly", - "kaleido==0.2.1", - "fire", - "pyyaml", - "BeautifulSoup4==4.11.*", - "prometheus_api_client", - "pydantic", - "opensearch-py", - "boto3", - "pandas", -] agentic_review = [ "httpx", "langchain-openai", @@ -118,10 +104,6 @@ exclude = ''' [tool.ruff] target-version = "py312" line-length = 100 -exclude = [ - "projects/matrix_benchmarking", - "projects/llm_d_legacy", -] [tool.ruff.lint] select = [ From 67fe34f9bd119dda0f243a1554800167f451f90e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:18:38 +0200 Subject: [PATCH 0879/1072] [llm_d_legacy] Remove --- projects/llm_d_legacy/orchestration/ci.py | 128 - .../llm_d_legacy/testing/command_args.yml.j2 | 8 - projects/llm_d_legacy/testing/config.yaml | 374 - .../epp-approximate-prefix-cache.yaml | 15 - .../epp-default-rhoai-3.4-ea.1.yaml | 29 - .../testing/epp-config/epp-pd.v0.4.yaml | 29 - .../testing/epp-config/epp-pd.v0.6.yaml | 38 - .../epp-config/epp-precise-prefix-cache.yaml | 26 - .../dashboards/container-overview.yaml | 2119 ------ .../dashboards/k8s-dashboard-starslcn.yaml | 6731 ----------------- .../dashboards/kubernetes-app-metrics.yaml | 1142 --- .../kubernetes-by-namespace-instance.yaml | 936 --- .../grafana/dashboards/llmd-vllm-wva.yaml | 6471 ---------------- .../testing/grafana/dashboards/nvidia.yaml | 841 -- .../testing/grafana/dashboards/vllm.yaml | 1627 ---- .../workload-variant-autoscaler.yaml | 1193 --- .../testing/grafana/datasource.yaml | 31 - .../testing/llmisvcs/llmisvc-pd.yaml | 161 - .../testing/llmisvcs/llmisvc-simple.yaml | 93 - projects/llm_d_legacy/testing/prepare_llmd.py | 804 -- .../testing/presets.d/presets.yaml | 105 - projects/llm_d_legacy/testing/test.py | 182 - projects/llm_d_legacy/testing/test_llmd.py | 1502 ---- projects/llm_d_legacy/toolbox/llmd.py | 136 - .../defaults/main/config.yml | 15 - .../llmd_capture_isvc_state/tasks/main.yml | 132 - .../vars/main/resources.yml | 1 - .../defaults/main/config.yml | 17 - .../toolbox/llmd_deploy_gateway/files/.keep | 0 .../toolbox/llmd_deploy_gateway/meta/main.yml | 3 - .../llmd_deploy_gateway/tasks/main.yml | 76 - .../templates/gateway.yaml.j2 | 14 - .../vars/main/resources.yml | 1 - .../defaults/main/config.yml | 20 - .../tasks/main.yml | 165 - .../vars/main/resources.yml | 1 - .../defaults/main/config.yml | 36 - .../tasks/main.yml | 146 - .../templates/copy_helper_pod.yaml.j2 | 35 - .../templates/guidellm_benchmark_job.yaml.j2 | 43 - .../templates/guidellm_benchmark_pvc.yaml.j2 | 11 - .../vars/main/resources.yml | 3 - .../defaults/main/config.yml | 48 - .../files/entrypoint.sh | 180 - .../storage_download_to_pvc/meta/main.yml | 2 - .../storage_download_to_pvc/tasks/main.yml | 163 - .../templates/pod.yml.j2 | 58 - .../templates/pvc.yml.j2 | 14 - .../vars/main/resources.yml | 4 - .../llmd_inference/analyze/__init__.py | 78 - .../llmd_inference/data/plots.yaml | 13 - .../llmd_inference/models/__init__.py | 2 - .../llmd_inference/models/kpi.py | 3 - .../llmd_inference/models/lts.py | 112 - .../llmd_inference/plotting/__init__.py | 19 - .../llmd_inference/plotting/error_report.py | 629 -- .../llmd_inference/plotting/prometheus.py | 12 - .../plotting/prometheus_reports.py | 244 - .../llmd_inference/plotting/report.py | 506 -- .../plotting/throughput_analysis.py | 1381 ---- .../plotting/throughput_comparisons.py | 565 -- .../llmd_inference/plotting/utils.py | 66 - .../llmd_inference/plotting/vllm_metrics.py | 369 - .../llmd_inference/requirements.txt | 4 - .../llmd_inference/store/__init__.py | 55 - .../llmd_inference/store/lts_parser.py | 96 - .../llmd_inference/store/parsers.py | 806 -- 67 files changed, 30869 deletions(-) delete mode 100755 projects/llm_d_legacy/orchestration/ci.py delete mode 100644 projects/llm_d_legacy/testing/command_args.yml.j2 delete mode 100644 projects/llm_d_legacy/testing/config.yaml delete mode 100644 projects/llm_d_legacy/testing/epp-config/epp-approximate-prefix-cache.yaml delete mode 100644 projects/llm_d_legacy/testing/epp-config/epp-default-rhoai-3.4-ea.1.yaml delete mode 100644 projects/llm_d_legacy/testing/epp-config/epp-pd.v0.4.yaml delete mode 100644 projects/llm_d_legacy/testing/epp-config/epp-pd.v0.6.yaml delete mode 100644 projects/llm_d_legacy/testing/epp-config/epp-precise-prefix-cache.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/container-overview.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/k8s-dashboard-starslcn.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-app-metrics.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-by-namespace-instance.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/llmd-vllm-wva.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/nvidia.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/vllm.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/dashboards/workload-variant-autoscaler.yaml delete mode 100644 projects/llm_d_legacy/testing/grafana/datasource.yaml delete mode 100644 projects/llm_d_legacy/testing/llmisvcs/llmisvc-pd.yaml delete mode 100644 projects/llm_d_legacy/testing/llmisvcs/llmisvc-simple.yaml delete mode 100644 projects/llm_d_legacy/testing/prepare_llmd.py delete mode 100644 projects/llm_d_legacy/testing/presets.d/presets.yaml delete mode 100755 projects/llm_d_legacy/testing/test.py delete mode 100644 projects/llm_d_legacy/testing/test_llmd.py delete mode 100644 projects/llm_d_legacy/toolbox/llmd.py delete mode 100644 projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/defaults/main/config.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/tasks/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/vars/main/resources.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/defaults/main/config.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/files/.keep delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/meta/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/tasks/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/templates/gateway.yaml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_gateway/vars/main/resources.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/defaults/main/config.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/tasks/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/vars/main/resources.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/defaults/main/config.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/tasks/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/copy_helper_pod.yaml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_job.yaml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_pvc.yaml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/vars/main/resources.yml delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/defaults/main/config.yml delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/files/entrypoint.sh delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/meta/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/tasks/main.yml delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pod.yml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pvc.yml.j2 delete mode 100644 projects/llm_d_legacy/toolbox/storage_download_to_pvc/vars/main/resources.yml delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/analyze/__init__.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/data/plots.yaml delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/models/__init__.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/models/kpi.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/models/lts.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/__init__.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/error_report.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus_reports.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/report.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_analysis.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_comparisons.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/utils.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/plotting/vllm_metrics.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/requirements.txt delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/store/__init__.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/store/lts_parser.py delete mode 100644 projects/llm_d_legacy/visualizations/llmd_inference/store/parsers.py diff --git a/projects/llm_d_legacy/orchestration/ci.py b/projects/llm_d_legacy/orchestration/ci.py deleted file mode 100755 index c9e2c1c67..000000000 --- a/projects/llm_d_legacy/orchestration/ci.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -""" -Jump CI Project CI Operations - -This is the JumpCI CI entrypoint. It's used to run TOPSAIL-ng remotely inside a VPN cluster""" - -import sys -import traceback -from pathlib import Path -import logging - -import click - -from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint -from projects.core.library import env -from projects.core.library import config as forge_config -from projects.core.library import ci as ci_lib -from projects.core.library.export import caliper_export_entrypoint -from projects.core.library.replot import caliper_replot_entrypoint -from projects.legacy.library import config -from projects.legacy.library import env as legacy_env -from projects.caliper.orchestration.export import run_from_orchestration_config -from projects.core.library import vault - -logger = logging.getLogger(__name__) - -# Add the testing directory to path for imports -testing_dir = Path(__file__).parent.parent / "testing" -if str(testing_dir) not in sys.path: - sys.path.insert(0, str(testing_dir)) - -# Import llm_d legacy testing functionality -try: - import prepare_llmd, test_llmd - import test as test_mod - -except ImportError as e: - raise RuntimeError(f"Legacy LLM_D testing functionality not available: {e}") from e - - -def log(message: str, level: str = "info"): - """Log message with project prefix.""" - project_name = "llm_d" - icon = {"info": "ℹ️", "success": "✅", "error": "❌", "warning": "⚠️"}.get(level, "ℹ️") - click.echo(f"{icon} [{project_name}] {message}") - - -@click.group(cls=ci_lib.HelpfulGroup) -@click.pass_context -@ci_lib.safe_ci_function -def main(ctx): - """Jump CI Project CI Operations for TOPSAIL-NG.""" - ctx.ensure_object(dict) - init() - -inited = False -def init(): - global inited - if inited: - return - inited = True - - env.init() - legacy_env.init() - - testing_dir = Path(__file__).parent.parent / "testing" - config.init(testing_dir) - forge_config.init(testing_dir) - presets = config.project.get_config("project.args") - for preset in presets: - config.project.apply_preset(preset) - - test_mod.init() - vault.disable_strict_validation() - vault.init(config.project.get_config("vaults")) - - -@main.command() -@click.pass_context -@ci_lib.safe_ci_entrypoint -def test(ctx): - """Test phase - Trigger the project's test method.""" - log("Starting test phase...") - - failed = test_llmd.test() - - sys.exit(1 if failed else 0) - - -main.add_command(caliper_export_entrypoint) -main.add_command(caliper_replot_entrypoint) - -def resolve_hardware_request(hardware_spec: dict): - """ - Resolve hardware requirements for FournosJob based on skeleton project configuration. - - This is a stub implementation. Update spec.hardware based on project configuration. - - Args: - hardware_spec: The current spec.hardware dict from the FournosJob. This object should be updated. - - """ - init() - - logger.info("Hardware resolution: stub implementation - no changes made") - - # Stub implementation - could be extended to: - # - Read hardware config from project config - # - Set hardware requirements based on workload needs - # - Handle different hardware profiles (GPU, CPU, memory requirements) - # - Example: return {"gpu": {"type": "nvidia-tesla-v100", "count": 1}, "memory": "32Gi"} - - return hardware_spec - -def list_vaults(): - init() - return config.project.get_config("vaults") - -main.add_command( - create_fournos_resolve_entrypoint( - vault_list_func=list_vaults, - hardware_resolver_func=resolve_hardware_request, - ) -) - - -if __name__ == "__main__": - main() diff --git a/projects/llm_d_legacy/testing/command_args.yml.j2 b/projects/llm_d_legacy/testing/command_args.yml.j2 deleted file mode 100644 index 0e41b43ab..000000000 --- a/projects/llm_d_legacy/testing/command_args.yml.j2 +++ /dev/null @@ -1,8 +0,0 @@ - -cluster reset_prometheus_db/uwm: - label: app.kubernetes.io/instance=user-workload,app.kubernetes.io/component=prometheus - namespace: openshift-user-workload-monitoring - -cluster dump_prometheus_db/uwm: - label: app.kubernetes.io/instance=user-workload,app.kubernetes.io/component=prometheus - namespace: openshift-user-workload-monitoring diff --git a/projects/llm_d_legacy/testing/config.yaml b/projects/llm_d_legacy/testing/config.yaml deleted file mode 100644 index dd4050ccf..000000000 --- a/projects/llm_d_legacy/testing/config.yaml +++ /dev/null @@ -1,374 +0,0 @@ -vaults: -- psap-forge-notifications -- psap-forge-mlflow-export - -project: - name: null - args: [] -ci_job: - name: null - fjob: null - cluster: null - exclusive: null - hardware: null - owner: null - -project: - name: null - args: [] - -ci_presets: - # name of the presets to apply, or null if no preset - name: null - # list of names of the presets to apply, or a single name, or null if no preset - names: null - local_config: null - to_apply: [] - - -clusters: - cleanup_on_exit: false - -security: - # Run containers as root user (affects both GuideLLM benchmarks and storage download operations) - run_as_root: false - -secrets: - dir: - name: psap-ods-secret - env_key: PSAP_ODS_SECRET_PATH - rhoai_rc: rhoai_rc.secret - hf_token: git-credentials - -prepare: - skip: false - namespace: - name: llm-d-project - - operators: - skip: true - list: - - name: "Red Hat Connectivity Link" - catalog: redhat-operators - operator: rhcl-operator - namespace: all - enabled: false - - - name: "OpenShift Cert Manager" - catalog: redhat-operators - operator: openshift-cert-manager-operator - namespace: openshift-cert-manager-operator - enabled: true - - - name: "Leader Worker Set" - catalog: redhat-operators - operator: leader-worker-set - namespace: openshift-lws - deploy_cr: true - enabled: true - - - name: "Node Feature Discovery" - catalog: redhat-operators - operator: nfd - namespace: openshift-nfd - deploy_cr: 1 - enabled: true - - - name: "NVIDIA GPU Operator" - catalog: certified-operators - operator: gpu-operator-certified - namespace: nvidia-gpu-operator - deploy_cr: true - enabled: true - - - name: "Grafana Operator" - catalog: community-operators - operator: grafana-operator - namespace: grafana-operator - enabled: true - extra_args: - all_namespaces: true - - cluster: - skip: false - nodes: - auto_scale: false - auto_scale_down_on_exit: false - instance_type: gx3-16x80x1l4 - count: 1 - - rhoai: - skip: false - image: "quay.io/rhoai/rhoai-fbc-fragment@sha256" - tag: "17970f248eb3b3df410a5a77da57cc893f0969d549088f76cd50165fa7576937" - channel: "beta" - datasciencecluster: - enable: "[kserve]" - extra_settings: '{"spec.components.kserve.rawDeploymentServiceConfig": "Headless"}' - - gateway: - skip: false - name: openshift-ai-inference # NOTE: Should not be changed for the time being - - grafana: - skip: true - namespace: grafana - datasources: - - grafana/datasource.yaml - dashboards_dir: grafana/dashboards - - monitoring: - skip: true - namespaces: - - "@prepare.namespace.name" - - gpu: - wait_for_readiness: false - - preload: - skip: true - extra_images: {} - node_selector_key: nvidia.com/gpu.present - node_selector_value: "true" - - pvc: - enabled: true - size: 2000Gi - name: storage - access_mode: ReadWriteOnce - storage_class: null - - model_downloader: - image: ghcr.io/opendatahub-io/rhaii-on-xks/kserve-storage-initializer:e6b5db0@sha256:b305264fe2211be2c6063500c4c11da79e8357af4b34dd8567b0d8e8dea7e1d4 - - cleanup: - skip: false - -deployment_profiles: - __default__: - tests.llmd.inference_service.yaml_file: llmisvc-simple.yaml - tests.llmd.inference_service.epp.deploy: "default" - tests.llmd.inference_service.epp.config_file: null # use the default config - - simple: - tests.llmd.inference_service.yaml_file: llmisvc-simple.yaml - tests.llmd.inference_service.epp.deploy: false - - intelligentrouting: - tests.llmd.inference_service.yaml_file: llmisvc-simple.yaml - - pd: - tests.llmd.inference_service.yaml_file: llmisvc-pd.yaml - tests.llmd.inference_service.epp.deploy: "default" - tests.llmd.inference_service.epp.config_file: null - -models: - facebook-opt-125m: - name: facebook/opt-125m - source: hf://facebook/opt-125m - resources: - cpu: 2 - memory: 8Gi - - llama3-1-8b: - name: RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic - uri: oci://registry.redhat.io/rhelai1/modelcar-llama-3-1-8b-instruct-fp8-dynamic:1.5 - # source: hf://RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8-dynamic - resources: {} - - llama3-3-70b: - name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - source: hf://RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - resources: - cpu: 4 - memory: 64Gi - - gpt-oss-120: - name: openai/gpt-oss-120b - source: hf://openai/gpt-oss-120b - resources: - cpu: 4 - memory: 64Gi - - granite4-tiny: - name: RedHatAI/granite-4.0-h-tiny-FP8-dynamic - source: hf://RedHatAI/granite-4.0-h-tiny-FP8-dynamic - resources: {} - - gpt-oss-20b: - name: openai/gpt-oss-20b - source: hf://openai/gpt-oss-20b - -tests: - llmd: - skip: false - skip_prepare: false - flavors: intelligentrouting - namespace: "@prepare.namespace.name" - - inference_service: - skip_deployment: false - name: llm-d - yaml_file: null # defined in the deployment_profile - image_pull_secrets: null - - epp: - deploy: "default" - config_file: null # use the default config - - do_simple_test: true - gateway: - name: gateway-external - model: llama3-3-70b - max_model_len: 40960 - metrics: - manual_capture: false - scheduler_servicemonitor_name: kserve-llm-isvc-scheduler - vllm_podmonitor_name: kserve-llm-isvc-vllm-engine - - # vLLM arguments (always applied) - vllm_args: - - "--disable-uvicorn-access-log" - - "--enable-prefix-caching" - - "--uvicorn-log-level=debug" - - "--trust-remote-code" - - "--gpu-memory-utilization=0.92" - - kueue: - enabled: false - prefix: "kueue.x-k8s.io/" - labels: - pod-group-name: llmisvc - managed: "true" - annotations: - queue-name: perf-gpu-queue - - # Extra properties to inject into the LLMISVC YAML using dotted-key notation - extra_properties: {} - - benchmarks: - guidellm: - enabled: true - name: guidellm-benchmark - load_shape_name: default - - image: - name: ghcr.io/vllm-project/guidellm - version: v0.6.0 - - rate: 1 # might be iterated, don't pass it in the args - - args: - backend_type: openai_http - rate_type: concurrent - max_seconds: 120 - max_requests: null - request_type: null - sample_requests: 20 - data: prompt_tokens=256,output_tokens=128 - - extra_args: {} - - - capture_prom: true - capture_prom_uwm: true - dry_mode: false - visualize: true - -export_artifacts: - enabled: false - -matbench: - enabled: true - preset: null - workload: projects.llm_d_legacy.visualizations.llmd_inference - config_file: plots.yaml - # directory to plot - lts: - generate: false - opensearch: - export: - enabled: false - enabled_on_replot: false - fail_test_on_fail: true - instance: smoke - index: topsail-llm-d-cpt - index_prefix: "" - build_counter_index: "topsail-llm-d-builds" # used to generate a unique ID for each build - regression_analyses: - enabled: false - enabled_on_replot: true - upload_lts_on_regression: true - # if the regression analyses fail, mark the test as failed - fail_test_on_regression: true - notification: - enabled: true - title: "llm-d CPT" - download: - mode: prefer_cache - url: null - -exec_list: - _only_: false - - pre_cleanup_ci: null - prepare_ci: null - test_ci: null - post_cleanup_ci: null - matbench_run: true - generate_plots_from_pr_args: true - - -caliper: - export: - from: null - verbose: false - dry_run: false - backend: - mlflow: - enabled: true - secrets: - vault: - name: psap-forge-mlflow-export - mlflow_secret: mlflow-secret.yaml - aws_secret: aws.credentials - - config: - # Same keys/values as docs/caliper/mlflow-config.example.yaml - experiment: topsail-llmd - run_name: - description: > - Caliper export from FORGE - source_script: topsail-llmd project - tags: {} - parameters: {} - metrics: {} - - replot: - url: null # Set via /replot.url directive - keep: false # Set to true to keep download directory after replotting - - postprocess: - enabled: true - artifacts_dir: null - plugin_module: projects.caliper.postprocess.guidellm.plugin - postprocess_config: null - parse: - enabled: true - no_cache: false - visualize: - enabled: true - output_dir: null - reports: [benchmark_summary, latency_breakdown] - report_group: null - - kpi: - enabled: false - artifacts_to_kpis: - enabled: false - output: kpis.json - - analyze: - enabled: false - historical_kpis: null - output: kpi_analyze.json diff --git a/projects/llm_d_legacy/testing/epp-config/epp-approximate-prefix-cache.yaml b/projects/llm_d_legacy/testing/epp-config/epp-approximate-prefix-cache.yaml deleted file mode 100644 index ced17965f..000000000 --- a/projects/llm_d_legacy/testing/epp-config/epp-approximate-prefix-cache.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -plugins: -- type: queue-scorer -- type: kv-cache-utilization-scorer -- type: prefix-cache-scorer -schedulingProfiles: -- name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 diff --git a/projects/llm_d_legacy/testing/epp-config/epp-default-rhoai-3.4-ea.1.yaml b/projects/llm_d_legacy/testing/epp-config/epp-default-rhoai-3.4-ea.1.yaml deleted file mode 100644 index 8c338b4fc..000000000 --- a/projects/llm_d_legacy/testing/epp-config/epp-default-rhoai-3.4-ea.1.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -plugins: - - type: prefill-header-handler - - type: prefill-filter - - type: decode-filter - - type: max-score-picker - - type: prefix-cache-scorer - - type: queue-scorer - - type: pd-profile-handler - parameters: - threshold: 0 -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: queue-scorer - weight: 1.0 - - pluginRef: prefix-cache-scorer - weight: 1.0 - - pluginRef: max-score-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: queue-scorer - weight: 1.0 - - pluginRef: prefix-cache-scorer - weight: 1.0 - - pluginRef: max-score-picker diff --git a/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.4.yaml b/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.4.yaml deleted file mode 100644 index a09575ff6..000000000 --- a/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.4.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -plugins: -- type: prefill-header-handler -- type: prefix-cache-scorer - parameters: - hashBlockSize: 5 - maxPrefixBlocksToMatch: 256 - lruCapacityPerServer: 31250 -- type: prefill-filter -- type: decode-filter -- type: max-score-picker -- type: pd-profile-handler - parameters: - threshold: 10 - hashBlockSize: 5 -schedulingProfiles: -- name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: max-score-picker - - pluginRef: prefix-cache-scorer - weight: 50 -- name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: max-score-picker - - pluginRef: prefix-cache-scorer - weight: 50 diff --git a/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.6.yaml b/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.6.yaml deleted file mode 100644 index b5719ae25..000000000 --- a/projects/llm_d_legacy/testing/epp-config/epp-pd.v0.6.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -featureGates: -- prepareDataPlugins -plugins: -- type: prefill-header-handler -- type: prefix-cache-scorer - parameters: - maxPrefixBlocksToMatch: 256 - lruCapacityPerServer: 31250 -- type: queue-scorer -- type: prefill-filter -- type: decode-filter -- type: max-score-picker -- type: prefix-based-pd-decider - parameters: - nonCachedTokens: 16 -- type: pd-profile-handler - parameters: - primaryPort: 0 - deciderPluginName: prefix-based-pd-decider -schedulingProfiles: -- name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: max-score-picker - - pluginRef: prefix-cache-scorer - weight: 2 - - pluginRef: queue-scorer - weight: 1 -- name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: max-score-picker - - pluginRef: prefix-cache-scorer - weight: 2 - - pluginRef: queue-scorer - weight: 1 diff --git a/projects/llm_d_legacy/testing/epp-config/epp-precise-prefix-cache.yaml b/projects/llm_d_legacy/testing/epp-config/epp-precise-prefix-cache.yaml deleted file mode 100644 index 651a4e4ec..000000000 --- a/projects/llm_d_legacy/testing/epp-config/epp-precise-prefix-cache.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: inference.networking.x-k8s.io/v1alpha1 -kind: EndpointPickerConfig -plugins: - - type: single-profile-handler - - type: precise-prefix-cache-scorer - parameters: - indexerConfig: - tokenProcessorConfig: - blockSize: 64 - hashSeed: "42" - tokenizersPoolConfig: - hf: - tokenizersCacheDir: "/tmp/tokenizers" - - type: kv-cache-utilization-scorer - - type: queue-scorer - - type: max-score-picker -schedulingProfiles: - - name: default - plugins: - - pluginRef: precise-prefix-cache-scorer - weight: 3.0 - - pluginRef: kv-cache-utilization-scorer - weight: 2.0 - - pluginRef: queue-scorer - weight: 2.0 - - pluginRef: max-score-picker diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/container-overview.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/container-overview.yaml deleted file mode 100644 index 74517a3cf..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/container-overview.yaml +++ /dev/null @@ -1,2119 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: container-overview - annotations: - name: Container Overview - originalUrl: https://grafana.com/grafana/dashboards/14201 -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "7.4.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": 14201, - "graphTooltip": 0, - "id": null, - "iteration": 1616074131050, - "links": [], - "panels": [ - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "title": "Status", - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "sum(kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Total Pods", - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": false - }, - "mappings": [ - { - "from": "", - "id": 1, - "text": "False", - "to": "", - "type": 1, - "value": "0" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Phase" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ready" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 4, - "y": 1 - }, - "id": 18, - "options": { - "frameIndex": 0, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "label_join((kube_pod_status_phase{phase!~\"Succeeded|Running\"} == 1) or (kube_pod_status_phase{phase=\"Running\"} == 1 and ON(namespace, pod) kube_pod_status_ready{condition=\"true\"} == 0), \"key\", \"/\", \"namespace\", \"pod\") and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "label_join((kube_pod_status_ready{condition=\"true\"} == 0), \"key\", \"/\", \"namespace\", \"pod\") and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Abnormal Pods", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "key" - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value #A": true, - "__name__": true, - "condition": true, - "instance": true, - "job": true, - "key": false, - "namespace": true, - "pod": true - }, - "indexByName": { - "Time": 1, - "Value #A": 8, - "Value #B": 10, - "__name__": 2, - "condition": 9, - "instance": 3, - "job": 4, - "key": 0, - "namespace": 5, - "phase": 7, - "pod": 6 - }, - "renameByName": { - "Time": "", - "Value #B": "Ready", - "condition": "", - "key": "Pod", - "namespace": "Namespace", - "phase": "Phase", - "pod": "Pod" - } - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "sum(kube_pod_container_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Running Containers", - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": null, - "filterable": false - }, - "mappings": [ - { - "from": "", - "id": 1, - "text": "False", - "to": "", - "type": 1, - "value": "0" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Restart (1h)" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Running" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 20, - "options": { - "frameIndex": 1, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "label_join((kube_pod_container_status_running == 0 or ON(namespace, pod, container) (increase(kube_pod_container_status_restarts_total[1h]) > 0)), \"key\", \"/\", \"namespace\", \"pod\") and ON(namespace, pod, container) kube_pod_container_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "expr": "label_join((increase(kube_pod_container_status_restarts_total[1h]) > 0), \"key\", \"/\", \"namespace\", \"pod\") and ON(namespace, pod, container) kube_pod_container_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - } - ], - "title": "Abnormal Containers", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "key" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "__name__": true, - "container 1": false, - "container 2": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "namespace": true, - "namespace 1": true, - "namespace 2": true, - "pod": true, - "pod 1": true, - "pod 2": true - }, - "indexByName": { - "Time 1": 4, - "Time 2": 10, - "Value #A": 2, - "Value #C": 3, - "__name__": 5, - "container 1": 1, - "container 2": 11, - "instance 1": 6, - "instance 2": 12, - "job 1": 7, - "job 2": 13, - "key": 0, - "namespace 1": 8, - "namespace 2": 14, - "pod 1": 9, - "pod 2": 15 - }, - "renameByName": { - "Time 1": "", - "Value #A": "Running", - "Value #C": "Restart (1h)", - "__name__": "", - "container 1": "Container", - "key": "Pod" - } - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 4 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "count((kube_pod_status_ready{condition=\"true\"} == 0) and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}) or vector(0)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Abnormal Pods", - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": {}, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 4 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "7.4.2", - "targets": [ - { - "expr": "count(kube_pod_container_status_ready == 0 and ON(namespace, pod, container) kube_pod_container_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"}) or vector(0)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Abnormal Containers", - "type": "stat" - }, - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 6, - "panels": [], - "title": "Resource", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_cpu_usage_seconds_total{container!=\"\"}[5m]) / ON(namespace, pod, container) (container_spec_cpu_quota/100000 > 0) and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage (Relative to Total)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 8 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, container_memory_working_set_bytes{container!=\"\"} / (container_spec_memory_limit_bytes > 0) and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage (Relative to Total)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 8 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, kubelet_volume_stats_used_bytes/kubelet_volume_stats_capacity_bytes * ON(namespace, persistentvolumeclaim) GROUP_LEFT(volume, pod) kube_pod_spec_volumes_persistentvolumeclaims_info and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Volume Usage (Relative to Total)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_cpu_usage_seconds_total{container!=\"\"}[5m]) and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage (Absolute)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "core", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 18 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, container_memory_working_set_bytes{container!=\"\"} and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage (Absolute)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 18 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, kubelet_volume_stats_used_bytes * ON(namespace, persistentvolumeclaim) GROUP_LEFT(volume, pod) kube_pod_spec_volumes_persistentvolumeclaims_info and ON(namespace, pod) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "", - "legendFormat": "{{ namespace }}/{{ pod }} - {{ container }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Volume Usage (Relative to Total)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "decimals": null, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 22, - "panels": [], - "title": "Network", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 29 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_network_receive_packets_total{pod!=\"\"}[5m]) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Packets Received", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "pps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 29 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, avg(rate(container_network_receive_bytes_total{pod!=\"\"}[5m])) without (id) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bytes Received", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 29 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": false, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_network_receive_packets_dropped_total{pod!=\"\"}[5m]) / (rate(container_network_receive_packets_total{pod!=\"\"}[5m]) > 0) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Packets Received Drop Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 38 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_network_transmit_packets_total{pod!=\"\"}[5m]) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Packet Transmitted", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "pps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 38 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, avg(rate(container_network_transmit_bytes_total{pod!=\"\"}[5m])) without (id) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bytes Transmitted", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 38 - }, - "hiddenSeries": false, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideZero": true, - "max": true, - "min": false, - "show": true, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk($TopK, rate(container_network_transmit_packets_dropped_total{pod!=\"\"}[5m]) / rate(container_network_transmit_packets_total{pod!=\"\"}[5m]) and ON(namespace, pod, container) kube_pod_info{created_by_kind=~\"$CreatedByKind\", created_by_name=~\"$CreatedByName\", namespace=~\"$Namespace\"})", - "interval": "$Step", - "legendFormat": "{{ namespace }}/{{ pod }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Packet Transmitted Drop Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1844", - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:1845", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 27, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "DS_PROMETHEUS", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [ ], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "kube_pod_info", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "Namespace", - "options": [], - "query": { - "query": "kube_pod_info", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/.*namespace=\\\"(.*?)\\\".*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "kube_pod_info", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Created By Kind", - "multi": true, - "name": "CreatedByKind", - "options": [], - "query": { - "query": "kube_pod_info", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/created_by_kind=\\\"(.*?)\\\"/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "kube_pod_info{namespace=~\"$Namespace\", created_by_kind=~\"$CreatedByKind\"}", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Created By Name", - "multi": true, - "name": "CreatedByName", - "options": [], - "query": { - "query": "kube_pod_info{namespace=~\"$Namespace\", created_by_kind=~\"$CreatedByKind\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/created_by_name=\\\"(.*?)\\\"/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": true, - "text": "30", - "value": "30" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "TopK", - "options": [ - { - "selected": false, - "text": "3", - "value": "3" - }, - { - "selected": false, - "text": "5", - "value": "5" - }, - { - "selected": false, - "text": "10", - "value": "10" - }, - { - "selected": false, - "text": "15", - "value": "15" - }, - { - "selected": true, - "text": "30", - "value": "30" - }, - { - "selected": false, - "text": "50", - "value": "50" - }, - { - "selected": false, - "text": "100", - "value": "100" - } - ], - "query": "3,5,10,15,30,50,100", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "auto": true, - "auto_count": 300, - "auto_min": "30s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_Step" - }, - "description": null, - "error": null, - "hide": 0, - "label": null, - "name": "Step", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_Step" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "30s,1m,2m,5m,10m,30m,1h", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h" - ] - }, - "timezone": "", - "title": "Container Overview", - "uid": "zIdVhMwGz", - "version": 4, - "description": "An Overview of the Pods and the Containers, filterable by namespace and parent controller." - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/k8s-dashboard-starslcn.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/k8s-dashboard-starslcn.yaml deleted file mode 100644 index 2d6c08b45..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/k8s-dashboard-starslcn.yaml +++ /dev/null @@ -1,6731 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: k8s-dashboard-starslcn - annotations: - name: K8S Dashboard (StarsL.cn) - originalUrl: https://grafana.com/grafana/dashboards/15661-k8s-dashboard-en-20250125/ -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "annotations": { - "list": [ - { - "$$hashKey": "object:247", - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "【English version】2025.01.25 Updates，kubernetesComprehensive display of resources！Includes K8S Overall Resource Overview、Microservices Resource Details、Pod Resource Details andK8SNetwork Bandwidth，Optimization metrics。https://grafana.com/orgs/starsliao/dashboards", - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 12, - "links": [ - { - "icon": "bolt", - "tags": [], - "targetBlank": true, - "title": "Update", - "tooltip": "See more dashboards", - "type": "link", - "url": "https://grafana.com/orgs/starsliao/dashboards" - }, - { - "$$hashKey": "object:831", - "icon": "question", - "tags": [ - "node_exporter" - ], - "targetBlank": true, - "title": "GitHub", - "tooltip": "View more dashboards", - "type": "link", - "url": "https://github.com/starsliao" - } - ], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 54, - "panels": [], - "title": "Node Resource Overview：Selected Nodes:【$Node】", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 0.8 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 44, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Utilization", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Request Rate", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Memory Limit Rate", - "refId": "B", - "step": 10 - } - ], - "title": "Node Memory Ratio", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 45, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Utilization Rate", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "CPU Request Rate", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "CPU Limit Rate", - "refId": "B" - } - ], - "title": "Node CPU Ratio", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of cluster nodes，Nodes POD Number of，NodesPODUpper Limit", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 1000 - }, - { - "color": "red", - "value": 2000 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 1 - }, - "id": 74, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "instant": true, - "interval": "", - "legendFormat": "Number of nodes", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Pod Number of nodes", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Upper limit Pod", - "refId": "C" - } - ], - "title": "Nodes with Pod", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-text" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Space" - }, - "properties": [ - { - "id": "custom.width", - "value": 59 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 21 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "SVC" - }, - "properties": [ - { - "id": "custom.width", - "value": 7 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Microservices" - }, - "properties": [ - { - "id": "custom.width", - "value": 4 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Configuration" - }, - "properties": [ - { - "id": "custom.width", - "value": 16 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Passwords" - }, - "properties": [ - { - "id": "custom.width", - "value": 33 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 11, - "y": 1 - }, - "id": 51, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Microservices" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}) by (namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(kube_service_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count by (namespace)({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(kube_configmap_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "configmap", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(kube_secret_info{origin_prometheus=~\"$origin_prometheus\"}) by(namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "secret", - "refId": "E" - } - ], - "title": "Namespace Resource Statistics", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "namespace" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true - }, - "includeByName": {}, - "indexByName": { - "Time 1": 2, - "Time 2": 4, - "Time 3": 6, - "Value #A": 3, - "Value #C": 5, - "Value #D": 1, - "namespace": 0 - }, - "renameByName": { - "Time 1": "", - "Time 2": "", - "Value #A": "Pod", - "Value #B": "Configuration", - "Value #C": "SVC", - "Value #D": "Microservices", - "Value #E": "Passwords", - "namespace": "Spaces" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 32, - "options": { - "legend": { - "calcs": [ - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "width": 200 - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Receive", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m]))*8", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Send", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "title": "$NameSpace：Network Overview（Associable nodes and namespaces）", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 2000000000000, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 100000000000 - }, - { - "color": "red", - "value": 2000000000000 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 71, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total Memory", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Limits", - "refId": "B" - } - ], - "title": "Node Memory Information", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 500, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "#EAB839", - "value": 500 - }, - { - "color": "red", - "value": 1000 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 72, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total Cores", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "Node CPU Number of cores", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "max": 8000000000000, - "min": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "#EAB839", - "value": 5000000000000 - }, - { - "color": "red", - "value": 10000000000000 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Utilization" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "max" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "thresholds", - "value": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "decimals", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 8, - "y": 5 - }, - "id": 73, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": false, - "sizing": "auto", - "text": {}, - "valueMode": "color" - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Utilization Rate", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Usage", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})", - "instant": true, - "interval": "", - "legendFormat": "Total", - "refId": "B" - } - ], - "title": "Node Storage Information", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Anomaly.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 88, - "maxPerRow": 2, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.1", - "repeat": "origin_prometheus", - "repeatDirection": "v", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count({__name__=~\"kube_deployment_metadata_generation|kube_daemonset_metadata_generation|kube_statefulset_metadata_generation\",origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Workload", - "range": false, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Total Pod", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key=~\"node.kubernetes.io.*\"})", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "{{key}}", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": true, - "legendFormat": "Total Nodes", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by(origin_prometheus)(kube_node_info{origin_prometheus=~\"$origin_prometheus\"}) - count by(origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", - "hide": false, - "instant": true, - "legendFormat": "Normal Node", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by(key,origin_prometheus)(kube_node_spec_taint{origin_prometheus=~\"$origin_prometheus\",key!~\"node.kubernetes.io.*\"})", - "hide": false, - "instant": true, - "legendFormat": "{{key}}", - "range": false, - "refId": "A" - } - ], - "title": "", - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "(node.kubernetes.io/)(.*)", - "renamePattern": "Abnormal:$2" - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total Memory" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 11 - }, - "id": 79, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Total Memory", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "Memory Usage【$Node】", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total Cores" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 11 - }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})", - "instant": false, - "interval": "", - "legendFormat": "Total Cores", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",id=\"/\",node=~\"^$Node$\"}[2m]))", - "instant": false, - "interval": "", - "legendFormat": "Usage", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "hide": true, - "instant": false, - "interval": "", - "legendFormat": "Requests", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})", - "hide": true, - "instant": true, - "interval": "", - "legendFormat": "Limit", - "refId": "B" - } - ], - "title": "CPU Used Cores【$Node】", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of Cluster Nodes，Nodes POD Number of，Nodes POD Upper Limit", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Upper Limit Pod" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Number of nodes" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.drawStyle", - "value": "points" - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.pointSize", - "value": 3 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 11 - }, - "id": 81, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "count(kube_node_info{origin_prometheus=~\"$origin_prometheus\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Number of nodes", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Pod Number", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Upper Limit Pod", - "refId": "C" - } - ], - "title": "Pod Number and nodes【$Node】", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Total number of cores.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#C4162A", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 75, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)*100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{node}}", - "refId": "I" - } - ], - "title": "$Node：Nodes CPU Breakdown", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 76, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)*100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{node}}", - "refId": "I" - } - ], - "title": "$Node：Node Memory Breakdown", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "series", - "axisLabel": "←Inflow/Outflow→", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Inflow.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 78, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{node}}", - "metric": "network", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum (irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\"}[2m]))by (node) *8", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflows:{{node}}", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "title": "$Node：Node Network Overview", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "CPU Limitations" - }, - "properties": [ - { - "id": "custom.width", - "value": 76 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 71 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Limits" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "custom.width", - "value": 85 - }, - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(Memory Usage|Total Memory|Memory Request|Memory Limit|Disk Usage|Disk Total)" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nodes" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 76 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Requests" - }, - "properties": [ - { - "id": "custom.width", - "value": 75 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "(CPUTotal|MemoryTotal|DiskTotal|PodLimit)" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PodCap" - }, - "properties": [ - { - "id": "custom.width", - "value": 66 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "CPU Core Usage$|Memory Usage$|Disk Usage$|Pod Number of" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Total/" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod Number of" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPUTotal Cores" - }, - "properties": [ - { - "id": "custom.width", - "value": 69 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory" - }, - "properties": [ - { - "id": "custom.width", - "value": 75 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Disk" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Core Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 102 - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 52, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Memory Usage%" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "count(kube_pod_info{origin_prometheus=~\"$origin_prometheus\",created_by_kind!~\"|Job\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "pod Number", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "kube_node_status_condition{origin_prometheus=~\"$origin_prometheus\",status=\"true\",node=~\"^$Node$\"} == 1", - "format": "table", - "hide": true, - "instant": true, - "interval": "", - "legendFormat": "Status", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m])) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"} - 0", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"}) by (node) - 0", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "K" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"}) by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Usage%", - "refId": "M" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Requests%", - "refId": "N" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Limit%", - "refId": "O" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container!=\"\",node=~\"^$Node$\"}[2m]))by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Usage%", - "refId": "P" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPU Requests%", - "refId": "Q" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",node=~\"^$Node$\"})by (node) / sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Memory Limit%", - "refId": "R" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node) / sum (container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",device=~\"^/dev/.*$\",id=\"/\",node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Disk Usage%", - "refId": "S" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_node_status_allocatable{origin_prometheus=~\"$origin_prometheus\",resource=\"pods\", unit=\"integer\",node=~\"^$Node$\"})by (node)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "PodCaps", - "refId": "T" - } - ], - "title": "$Node：Node Information Detail", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 18": true, - "Time 19": true, - "Time 2": true, - "Time 20": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #B": true, - "Value #E": false, - "Value #F": false, - "__name__": true, - "app_kubernetes_io_name": true, - "app_kubernetes_io_name 1": true, - "app_kubernetes_io_name 2": true, - "app_kubernetes_io_name 3": true, - "app_kubernetes_io_version": true, - "app_kubernetes_io_version 1": true, - "app_kubernetes_io_version 2": true, - "app_kubernetes_io_version 3": true, - "condition": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "k8s_namespace": true, - "k8s_namespace 1": true, - "k8s_namespace 2": true, - "k8s_namespace 3": true, - "k8s_sname": true, - "k8s_sname 1": true, - "k8s_sname 2": true, - "k8s_sname 3": true, - "origin_prometheus": true, - "origin_prometheus 1": true, - "origin_prometheus 2": true, - "origin_prometheus 3": true, - "resource": true, - "status": true, - "unit": true - }, - "includeByName": {}, - "indexByName": { - "Time": 22, - "Value #A": 2, - "Value #C": 6, - "Value #D": 8, - "Value #E": 16, - "Value #F": 17, - "Value #G": 18, - "Value #H": 19, - "Value #I": 7, - "Value #J": 9, - "Value #K": 11, - "Value #L": 10, - "Value #M": 4, - "Value #N": 13, - "Value #O": 15, - "Value #P": 3, - "Value #Q": 12, - "Value #R": 14, - "Value #S": 5, - "Value #T": 1, - "instance": 23, - "job": 24, - "node": 0, - "origin_prometheus": 25, - "resource": 20, - "unit": 21 - }, - "renameByName": { - "Value #A": "Pod Number of", - "Value #C": "CPU Total Cores", - "Value #D": "Total Memory", - "Value #E": "CPU Requests", - "Value #F": "CPU Limit", - "Value #G": "Memory Requests", - "Value #H": "Memory Limit", - "Value #I": "CPU Core Usage", - "Value #J": "Memory Usage", - "Value #K": "Disk Usage", - "Value #L": "Disk Total", - "Value #M": "Memory Usage%", - "Value #N": "Memory Requests%", - "Value #O": "Memory Limit%", - "Value #P": "CPU Usage%", - "Value #Q": "CPU Requests%", - "Value #R": "CPU Limit%", - "Value #S": "Disk Usage%", - "Value #T": "Pod Limit", - "condition": "Status", - "node": "Node" - } - } - }, - { - "id": "filterFieldsByName", - "options": {} - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "color-background" - }, - "inspect": false - }, - "decimals": 0, - "links": [], - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 0 - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "special" - } - ], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Utilization" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "custom.width", - "value": 54 - }, - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - }, - { - "id": "decimals" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 75, - "result": { - "color": "semi-dark-red", - "index": 0 - }, - "to": 110 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Mounts Pod Number of" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.width", - "value": 59 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "PVC" - }, - "properties": [ - { - "id": "custom.width", - "value": 94 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 57 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "custom.width", - "value": 54 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 33 - }, - "id": 92, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Usage Rate" - } - ] - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}) /(min by (namespace,persistentvolumeclaim) (kubelet_volume_stats_available_bytes{origin_prometheus=~\"$origin_prometheus\"}) + max by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{origin_prometheus=~\"$origin_prometheus\"}))*100", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}:{{ persistentvolumeclaim }}", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "C", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count by (namespace,persistentvolumeclaim)(kube_pod_spec_volumes_persistentvolumeclaims_info{origin_prometheus=~\"$origin_prometheus\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "container_memory_usage:sort_desc", - "range": false, - "refId": "D", - "step": 10 - } - ], - "title": "PVC Storage Usage", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value #A": "Usage", - "Value #B": "Total", - "Value #C": "Usage Rate", - "Value #D": "Mount Pod Number", - "namespace": "Namespaces", - "persistentvolumeclaim": "PVC" - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 6, - "y": 33 - }, - "id": 86, - "options": { - "legend": { - "calcs": [ - "last", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}[2m])) by (namespace)>0.5", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ namespace }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Namespaces CPU Usage kernel(>0.5)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 33 - }, - "id": 85, - "options": { - "legend": { - "calcs": [ - "last", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container !=\"\",container!=\"POD\"}) by (namespace) > 1*1024*1024*1024", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace} {{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Namespaces WSS Memory Usage (>1G)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 49, - "panels": [], - "title": "Pod Resource Overview：SelectedPod:【$Pod】", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "displayName": "", - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 207 - }, - { - "id": "custom.align", - "value": "right" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cores Used" - }, - "properties": [ - { - "id": "custom.width", - "value": 71 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Reboot" - }, - "properties": [ - { - "id": "custom.width", - "value": 38 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "decimals" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.width", - "value": 55 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Restrictions" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "color-background" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Nodes" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "Cores in use$|WSS$|RSS$|Survival|Inflow|Outflow" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Container Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 57 - }, - { - "id": "custom.align", - "value": "left" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Survival" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.width", - "value": 80 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Used cores" - }, - "properties": [ - { - "id": "custom.width", - "value": 62 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Limits" - }, - "properties": [ - { - "id": "custom.width", - "value": 58 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 68 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Memory Requirements" - }, - "properties": [ - { - "id": "custom.width", - "value": 88 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "WSS$|RSS$|Memory Requirements$|Memory Limit$|Disk.*$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WSS" - }, - "properties": [ - { - "id": "custom.width", - "value": 81 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "RSS" - }, - "properties": [ - { - "id": "custom.width", - "value": 74 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "CPU Requirements" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Limitations" - }, - "properties": [ - { - "id": "custom.width", - "value": 83 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 10737418240 - }, - { - "color": "red", - "value": 16106127360 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WSS%" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/Inflow|Streaming Out/" - }, - "properties": [ - { - "id": "unit", - "value": "binbps" - }, - { - "id": "custom.width", - "value": 80 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 47, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "WSS%" - } - ] - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod,node,namespace)) ", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "CPUKernel Usage", - "refId": "Q" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "wss%", - "refId": "I" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "wss", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "rss%", - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "rss", - "refId": "K" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "kube_pod_container_status_restarts_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"} * on (pod) group_left(node) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "time() - kube_pod_created{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"} * on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "R" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "S" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "T" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "refId": "U" - } - ], - "title": "$Node：Pod Resource Detail(Associable Nodes)", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #G": false, - "__name__": true, - "app_kubernetes_io_name": true, - "app_kubernetes_io_name 1": true, - "app_kubernetes_io_name 2": true, - "app_kubernetes_io_version": true, - "app_kubernetes_io_version 1": true, - "app_kubernetes_io_version 2": true, - "container 1": true, - "container 10": true, - "container 11": true, - "container 12": true, - "container 2": true, - "container 3": true, - "container 4": true, - "container 5": true, - "container 6": true, - "container 7": true, - "container 8": true, - "container 9": true, - "created_by_kind": true, - "created_by_name": true, - "host_ip": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "k8s_namespace": true, - "k8s_namespace 1": true, - "k8s_namespace 2": true, - "k8s_sname": true, - "k8s_sname 1": true, - "k8s_sname 2": true, - "namespace": false, - "namespace 1": true, - "namespace 10": true, - "namespace 11": true, - "namespace 12": false, - "namespace 2": true, - "namespace 3": true, - "namespace 4": true, - "namespace 5": true, - "namespace 6": true, - "namespace 7": true, - "namespace 8": true, - "namespace 9": true, - "node 1": true, - "node 10": true, - "node 11": false, - "node 12": true, - "node 2": true, - "node 3": true, - "node 4": true, - "node 5": true, - "node 6": true, - "node 7": true, - "node 8": true, - "node 9": true, - "origin_prometheus": true, - "origin_prometheus 1": true, - "origin_prometheus 2": true, - "phase": true, - "pod_ip": true, - "priority_class": true, - "uid": true - }, - "includeByName": {}, - "indexByName": { - "Time": 21, - "Value #A": 4, - "Value #B": 16, - "Value #C": 7, - "Value #D": 10, - "Value #E": 17, - "Value #F": 9, - "Value #G": 23, - "Value #H": 14, - "Value #I": 5, - "Value #J": 13, - "Value #K": 11, - "Value #L": 6, - "Value #M": 24, - "Value #N": 25, - "Value #O": 26, - "Value #P": 27, - "Value #Q": 8, - "Value #R": 15, - "Value #S": 12, - "container": 2, - "instance": 18, - "ip": 28, - "job": 19, - "namespace": 1, - "node": 0, - "origin_prometheus": 20, - "pod": 3, - "uid": 22 - }, - "renameByName": { - "Value #A": "CPU%", - "Value #B": "CPU Requirements", - "Value #C": "CPU Limitations", - "Value #D": "WSS", - "Value #E": "Memory Requirements", - "Value #F": "Memory Limit", - "Value #H": "Reboot", - "Value #I": "WSS%", - "Value #J": "Disk Usage", - "Value #K": "RSS", - "Value #L": "RSS%", - "Value #M": "Heap Memory", - "Value #N": "maxHeap", - "Value #O": "Non-Heap", - "Value #P": "maxNon-Heap", - "Value #Q": "Using Cores", - "Value #R": "Survival", - "Value #S": "Disk Limit", - "Value #T": "Inflow", - "Value #U": "Streaming Out", - "container": "Container Name", - "instance": "", - "ip": "POD IP", - "namespace": "Namespace", - "namespace 1": "", - "namespace 12": "Namespace", - "node": "Nodes", - "node 1": "", - "node 11": "Node", - "pod": "Pod Name", - "priority_class": "" - } - } - }, - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Node", - "Namespace", - "Container Name", - "Pod Name", - "CPU%", - "WSS%", - "RSS%", - "CPU Restrictions", - "Cores Used", - "Memory Limit", - "WSS", - "RSS", - "Disk Limit", - "Disk Usage", - "Reboot", - "CPUDemand", - "Memory Requirements", - "Inflow", - "Outflow", - "Survival" - ] - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 50 - }, - "id": 58, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod) / (max(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}/100000) by (container, pod)) * 100", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ pod }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Pod Containers CPU Utilization (Maximum100%Associable Nodes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 50 - }, - "id": 27, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod)/ max(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "(cass_jvm_heap{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) / (cass_jvm_heap_max{service=~\"$Container\"} * on (pod_ip) group_right(service) kube_pod_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\"}) * 100", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Heap：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "refId": "C", - "step": 10 - } - ], - "title": "Pod Container Memory Usage (Associatable Nodes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 50 - }, - "id": 77, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(max(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{ pod}}", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(max(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(pod) *8", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflow:{{ pod}}", - "metric": "network", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "network", - "range": true, - "refId": "C", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info) by(pod) *8", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "__auto", - "metric": "network", - "range": true, - "refId": "D", - "step": 10 - } - ], - "title": "Pod Network bandwidth per second (Associable Nodes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 59 - }, - "id": 82, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (container, pod,node,namespace)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Usage：{{ pod }}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Pod CPU Limitations：{{ container}}", - "metric": "container_cpu", - "refId": "B", - "step": 10 - } - ], - "title": "Pod Containers CPU Core Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*Limitations/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 59 - }, - "id": 84, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(max(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",pod=~\"$Pod\",container =~\"$Container\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container,pod,node,namespace)) by(container)", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "PodMemory Limits：{{ container}}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - } - ], - "title": "Pod Containers WSS Memory Usage (Associable Nodes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 59 - }, - "id": 83, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "max (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",container =~\"$Container\",container !=\"\",container!=\"POD\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}) by (container, pod,node,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ pod }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Pod Containers RSS Memory Usage (Associable Nodes)", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 61, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*%" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge", - "valueDisplayMode": "color" - } - }, - { - "id": "max", - "value": 1 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "color", - "value": { - "mode": "continuous-GrYlRd" - } - }, - { - "id": "decimals", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Memory Usage$|.*Memory Limits$|.*Memory Requirements$|.*Disk Usage$|.*Disk Limits$" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Namespace" - }, - "properties": [ - { - "id": "custom.width", - "value": 92 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Container Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 187 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Core Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pod" - }, - "properties": [ - { - "id": "custom.width", - "value": 44 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average CPU Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 116 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average RSS Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 141 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average WSS Memory Usage%" - }, - "properties": [ - { - "id": "custom.width", - "value": 165 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 86 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 86 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Limit$/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*Memory Usage$|.*Core Usage$/" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total RSS Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 107 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total WSS Memory Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 113 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average Disk Usage" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "orange", - "value": 10737418240 - }, - { - "color": "red", - "value": 16106127360 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Average Disk Limit" - }, - "properties": [ - { - "id": "custom.width", - "value": 96 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total CPU Demand" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Memory Demand" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 87, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Average WSS Memory Usage%" - } - ] - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container))", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Total Cores Used", - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "I" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Average Memory%(RSS)", - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Total Memory Usage (RSS) ", - "refId": "K" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(container) (kube_pod_container_resource_requests{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"memory\", unit=\"byte\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(container_fs_usage_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "J" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(container_fs_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "M" - } - ], - "title": "Microservices (Container Name) Resource Statistics", - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true - }, - "includeByName": {}, - "indexByName": { - "Time": 15, - "Value #A": 3, - "Value #B": 13, - "Value #C": 6, - "Value #D": 9, - "Value #E": 14, - "Value #F": 8, - "Value #G": 2, - "Value #H": 5, - "Value #I": 4, - "Value #J": 12, - "Value #K": 10, - "Value #L": 7, - "Value #M": 11, - "container": 1, - "namespace": 0 - }, - "renameByName": { - "Time 1": "", - "Value #A": "Average CPU Usage%", - "Value #B": "Total CPU Demand", - "Value #C": "Total CPU Restrictions", - "Value #D": "Total WSS Memory Usage", - "Value #E": "Total Memory Requirements", - "Value #F": "Total Memory Limit", - "Value #G": "Pod", - "Value #H": "Average RSS Memory Usage%", - "Value #I": "Average WSS Memory Usage%", - "Value #J": "Average Disk Usage", - "Value #K": "Total RSS Memory Use", - "Value #L": "Total CPU Core Usage", - "Value #M": "Average Disk Limit", - "container": "Container Name", - "namespace": "Namespace" - } - } - }, - { - "id": "filterFieldsByName", - "options": {} - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container) / (sum(container_spec_cpu_quota{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}/100000) by (container)) * 100", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ container}}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Microservices (Container Name) Average CPU Usage (Maximum100%)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 89, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)/ sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container) * 100", - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "refId": "B", - "step": 10 - } - ], - "title": "Microservice (Container Name) Average Memory Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binbps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 16, - "options": { - "legend": { - "calcs": [ - "mean", - "last", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(sum(irate(container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Inflow:{{ container }}", - "metric": "network", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(sum(irate(container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",node=~\"^$Node$\",namespace=~\"$NameSpace\"}[2m])) by (pod)* on(pod) group_right kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container =~\"$Container\"}) by(container) *8", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Outflow:{{ container }}", - "metric": "network", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (rate (container_network_receive_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "-> {{ pod }}", - "metric": "network", - "refId": "C", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "- sum (rate (container_network_transmit_bytes_total{origin_prometheus=~\"$origin_prometheus\",pod=~\"$Pod\",image!=\"\",name=~\"^k8s_.*\",node=~\"^$Node$\",namespace=~\"$NameSpace\",pod=~\".*$Container.*\"}[2m])) by (pod)", - "hide": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "<- {{ pod }}", - "metric": "network", - "refId": "D", - "step": 10 - } - ], - "title": "Microservice (Container Name) Network bandwidth per second (Associable Nodes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/CPU Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 91, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(kube_pod_container_resource_limits{origin_prometheus=~\"$origin_prometheus\",resource=\"cpu\", unit=\"core\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Limitations：{{ container}}", - "metric": "container_cpu", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(irate(container_cpu_usage_seconds_total{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}[2m])) by (container)", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "CPU Core Usage：{{ container}}", - "metric": "container_cpu", - "refId": "B", - "step": 10 - } - ], - "title": "Microservices (Container Name) Overall CPU Cores Used", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Memory Limit.*/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 90, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (container_memory_working_set_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "WSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(container_spec_memory_limit_bytes{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Memory Limit：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "B", - "step": 10 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (container_memory_rss{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by (container)", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "RSS：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "C", - "step": 10 - } - ], - "title": "Microservices (Container Name) Overall Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 59, - "options": { - "legend": { - "calcs": [ - "max", - "last", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "count(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",container =~\"$Container\",container !=\"\",container!=\"POD\",namespace=~\"$NameSpace\"}) by(container,namespace)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}：{{ container }}", - "metric": "container_memory_usage:sort_desc", - "range": true, - "refId": "A", - "step": 10 - } - ], - "title": "Microservice (Container Name) Pod Number", - "type": "timeseries" - } - ], - "title": "Microservices (Container Name) Resource Overview：Selected Microservices:【$Container】", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 41, - "tags": [ - "Prometheus", - "Kubernetes" - ], - "templating": { - "list": [ - { - "current": { - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(kube_node_info,origin_prometheus)", - "includeAll": false, - "label": "K8S", - "name": "origin_prometheus", - "options": [], - "query": { - "query": "label_values(kube_node_info,origin_prometheus)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", - "includeAll": true, - "label": "Nodes", - "name": "Node", - "options": [], - "query": { - "query": "label_values(kube_node_info{origin_prometheus=~\"$origin_prometheus\"},node)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", - "includeAll": true, - "label": "Namespace", - "name": "NameSpace", - "options": [], - "query": { - "query": "label_values(kube_namespace_created{origin_prometheus=~\"$origin_prometheus\"},namespace)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", - "includeAll": true, - "label": "Microservice (Container Name)", - "name": "Container", - "options": [], - "query": { - "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\"},container)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", - "includeAll": true, - "label": "Pod", - "name": "Pod", - "options": [], - "query": { - "query": "label_values(kube_pod_container_info{origin_prometheus=~\"$origin_prometheus\",namespace=~\"$NameSpace\",container=~\"$Container\"},pod)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 5, - "type": "query" - }, - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "30s", - "1m", - "5m", - "15m", - "30m", - "1h" - ] - }, - "timezone": "browser", - "title": "K8S Dashboard (StarsL.cn)", - "uid": "4460b1f0-c7e3-4774-b393-d9117cdb97a8", - "version": 1 - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-app-metrics.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-app-metrics.yaml deleted file mode 100644 index b30de6827..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-app-metrics.yaml +++ /dev/null @@ -1,1142 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: kubernetes-app-metrics - annotations: - name: Kubernetes App Metrics - originalLink: https://grafana.com/grafana/dashboards/1471 -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "After selecting your namespace and container you get a wealth of metrics like request rate, error rate, response times, pod count, cpu and memory usage. You can view cpu and memory usage in a variety of ways, compared to the limit, compared to the request, per pod, average per pod, etc.", - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 16, - "panels": [], - "title": "Request rate", - "type": "row" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(http_requests_total{app=\"$container\", handler!=\"prometheus\", kubernetes_namespace=\"$namespace\"}[30s])) by (kubernetes_namespace,app,code)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "native | {{code}}", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(nginx_http_requests_total{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (kubernetes_namespace,app,status)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "nginx | {{status}}", - "refId": "B", - "step": 10 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(haproxy_backend_http_responses_total{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,code)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "haproxy | {{code}}", - "refId": "C", - "step": 10 - } - ], - "title": "Request rate", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(haproxy_backend_http_responses_total{app=\"$container\", kubernetes_namespace=\"$namespace\",code=\"5xx\"}[30s])) by (app,kubernetes_namespace) / sum(irate(haproxy_backend_http_responses_total{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "haproxy", - "refId": "A", - "step": 20 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(http_requests_total{app=\"$container\", handler!=\"prometheus\", kubernetes_namespace=\"$namespace\", code=~\"5[0-9]+\"}[30s])) by (kubernetes_namespace,app) / sum(irate(http_requests_total{app=\"$container\", handler!=\"prometheus\", kubernetes_namespace=\"$namespace\"}[30s])) by (kubernetes_namespace,app)", - "intervalFactor": 2, - "legendFormat": "native", - "refId": "B", - "step": 20 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(nginx_http_requests_total{app=\"$container\", kubernetes_namespace=\"$namespace\", status=~\"5[0-9]+\"}[30s])) by (kubernetes_namespace,app) / sum(irate(nginx_http_requests_total{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (kubernetes_namespace,app)", - "intervalFactor": 2, - "legendFormat": "nginx", - "refId": "C", - "step": 20 - } - ], - "title": "Error rate", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 17, - "panels": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 5, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "intervalFactor": 1, - "legendFormat": "native | 0.99", - "refId": "A", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.90, sum(rate(http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "intervalFactor": 1, - "legendFormat": "native | 0.90", - "refId": "B", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, sum(rate(http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "native | 0.50", - "refId": "C", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "intervalFactor": 1, - "legendFormat": "nginx | 0.99", - "refId": "D", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.9, sum(rate(nginx_http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "intervalFactor": 1, - "legendFormat": "nginx | 0.90", - "refId": "E", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.5, sum(rate(nginx_http_request_duration_seconds_bucket{app=\"$container\", kubernetes_namespace=\"$namespace\"}[30s])) by (app,kubernetes_namespace,le))", - "intervalFactor": 1, - "legendFormat": "nginx | 0.50", - "refId": "F", - "step": 1 - } - ], - "title": "Response time percentiles", - "type": "timeseries" - } - ], - "title": "Response time", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 18, - "panels": [], - "title": "Pod count", - "type": "row" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (pod_name))", - "interval": "", - "intervalFactor": 1, - "legendFormat": "pods", - "refId": "A", - "step": 5 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (kubernetes_io_hostname))", - "interval": "", - "intervalFactor": 2, - "legendFormat": "hosts", - "refId": "B", - "step": 10 - } - ], - "title": "Number of pods", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 19, - "panels": [], - "title": "Usage relative to request", - "type": "row" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "elasticsearch-logging-data-20170207a (logging) - system" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#BF1B00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "elasticsearch-logging-data-20170207a (logging) - user" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#508642", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.1.0", - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_system_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (namespace,container) / sum(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"} / 1024) by (namespace,container)", - "intervalFactor": 2, - "legendFormat": "system", - "refId": "C", - "step": 10 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_user_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (namespace,container) / sum(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"} / 1024) by (namespace,container)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "user", - "refId": "B", - "step": 10 - } - ], - "title": "Cpu usage (relative to request)", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 20, - "panels": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 10, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_usage_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (namespace,container) / sum(container_spec_cpu_quota{container=\"$container\", namespace=\"$namespace\"} / container_spec_cpu_period{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "actual", - "metric": "", - "refId": "A", - "step": 1 - } - ], - "title": "Cpu usage (relative to limit)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 11, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / sum(container_spec_memory_limit_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "actual", - "refId": "A", - "step": 1 - } - ], - "title": "Memory usage (relative to limit)", - "type": "timeseries" - } - ], - "title": "Usage relative to limit", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 21, - "panels": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 13, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_usage_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (id,pod_name)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod_name}}", - "refId": "A", - "step": 2 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_quota{container=\"$container\", namespace=\"$namespace\"} / container_spec_cpu_period{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "intervalFactor": 2, - "legendFormat": "limit", - "refId": "B", - "step": 2 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"} / 1024) by (namespace,container) / count(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "intervalFactor": 2, - "legendFormat": "request", - "refId": "C", - "step": 2 - } - ], - "title": "Cpu usage (per pod)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 14, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (id,pod_name)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod_name}}", - "refId": "A", - "step": 2 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_memory_limit_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (container,namespace)", - "intervalFactor": 2, - "legendFormat": "limit", - "refId": "B", - "step": 2 - } - ], - "title": "Memory usage (per pod)", - "type": "timeseries" - } - ], - "title": "Usage per pod", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 22, - "panels": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 8, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_usage_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "interval": "", - "intervalFactor": 1, - "legendFormat": "actual", - "refId": "A", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_quota{container=\"$container\", namespace=\"$namespace\"} / container_spec_cpu_period{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "interval": "", - "intervalFactor": 1, - "legendFormat": "limit", - "refId": "B", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"} / 1024) by (namespace,container) / count(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "interval": "", - "intervalFactor": 1, - "legendFormat": "request", - "refId": "C", - "step": 1 - } - ], - "title": "Cpu usage (avg per pod)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 9, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "intervalFactor": 1, - "legendFormat": "actual", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_memory_limit_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) / count(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container) ", - "interval": "", - "intervalFactor": 1, - "legendFormat": "limit", - "refId": "B", - "step": 1 - } - ], - "title": "Memory usage (avg per pod)", - "type": "timeseries" - } - ], - "title": "Usage per pod (average)", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 23, - "panels": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 1, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(container_cpu_usage_seconds_total{container=\"$container\", namespace=\"$namespace\"}[30s])) by (namespace,container)", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "actual", - "metric": "", - "refId": "A", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_quota{container=\"$container\", namespace=\"$namespace\"} / container_spec_cpu_period{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container)", - "intervalFactor": 1, - "legendFormat": "limit", - "refId": "B", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_cpu_shares{container=\"$container\", namespace=\"$namespace\"} / 1024) by (namespace,container) ", - "intervalFactor": 1, - "legendFormat": "request", - "refId": "C", - "step": 1 - } - ], - "title": "Cpu usage (total)", - "type": "timeseries" - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 2, - "options": {}, - "targets": [ - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_usage_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container)", - "interval": "", - "intervalFactor": 1, - "legendFormat": "actual", - "refId": "A", - "step": 1 - }, - { - "datasource": { - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_spec_memory_limit_bytes{container=\"$container\", namespace=\"$namespace\"}) by (namespace,container)", - "intervalFactor": 1, - "legendFormat": "limit", - "refId": "B", - "step": 1 - } - ], - "title": "Memory usage (total)", - "type": "timeseries" - } - ], - "title": "Usage total", - "type": "row" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 41, - "tags": [], - "templating": { - "list": [ - { - "allValue": ".+", - "current": { - "text": "", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "includeAll": false, - "name": "namespace", - "options": [], - "query": "label_values(container_memory_usage_bytes{namespace=~\".+\",container!=\"POD\"},namespace)", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "allValue": ".+", - "current": { - "text": "", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "includeAll": false, - "name": "container", - "options": [], - "query": "label_values(container_memory_usage_bytes{namespace=~\"$namespace\",container!=\"POD\"},container)", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Kubernetes App Metrics", - "uid": "4885d1af-3300-430d-a132-b05c6fe3ab8f", - "version": 1 - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-by-namespace-instance.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-by-namespace-instance.yaml deleted file mode 100644 index a2f838d84..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/kubernetes-by-namespace-instance.yaml +++ /dev/null @@ -1,936 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - annotations: - name: Kubernetes monitoring by namespace and instance - originalUrl: https://grafana.com/grafana/dashboards/15826 - name: kubernetes-by-namespace-instance -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "__inputs": [], - "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.3.4" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Monitor a Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Disk usage and number of nodes. As a individual and with the posibility of filter by namespace: POD CPU / Memory / Network usage and PVC usage.", - "editable": false, - "fiscalYearStartMonth": 0, - "gnetId": 15826, - "graphTooltip": 0, - "id": null, - "iteration": 1646896987307, - "links": [], - "liveNow": false, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 65 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 7, - "x": 0, - "y": 0 - }, - "id": 4, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "(sum(node_memory_MemTotal_bytes{kubernetes_node=~\"$instance\"}) - sum(node_memory_MemFree_bytes{kubernetes_node=~\"$instance\"}+node_memory_Buffers_bytes{kubernetes_node=~\"$instance\"}+node_memory_Cached_bytes{kubernetes_node=~\"$instance\"}) ) / sum(node_memory_MemTotal_bytes{kubernetes_node=~\"$instance\"}) * 100", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cluster memory usage", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 65 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 7, - "y": 0 - }, - "id": 6, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "sum(sum by (container_name)( rate(container_cpu_usage_seconds_total{image!=\"\", kubernetes_io_hostname=~\"$instance\"}[5m]) )) / count(node_cpu_seconds_total{mode=\"system\", kubernetes_node=~\"$instance\"}) * 100", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cluster CPU usage", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 65 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 13, - "y": 0 - }, - "id": 7, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"/\",kubernetes_node=~\"$instance\"}) - sum(node_filesystem_free_bytes{mountpoint=\"/\",kubernetes_node=~\"$instance\"}) ) / sum(node_filesystem_size_bytes{mountpoint=\"/\",kubernetes_node=~\"$instance\"}) * 100", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 10 - } - ], - "title": "Cluster Filesystem usage", - "type": "gauge" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "displayName": "nodes", - "mappings": [], - "max": 6, - "min": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 4 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "count (count by (node) (kube_node_info))", - "interval": "", - "legendFormat": "#Nodes", - "refId": "A" - } - ], - "title": "Number of nodes", - "type": "stat" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "datadir-graylog-mongodb-0" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#56A64B", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "1 / kubelet_volume_stats_capacity_bytes * kubelet_volume_stats_used_bytes{namespace=~\"$namespace\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "title": "PVC usage", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=\"\",namespace=~\"^$namespace$\",kubernetes_io_hostname=~\"$instance\"}[5m] ) )", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "pod_cpu", - "refId": "A", - "step": 10 - } - ], - "title": "Pod CPU usage", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 2, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "sum by (pod_name)( rate(container_memory_usage_bytes{image!=\"\",namespace=~\"^$namespace$\",kubernetes_io_hostname=~\"$instance\"}[5m] ) )", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "pod_memory_usage:sort_desc", - "refId": "A", - "step": 10 - } - ], - "title": "Pod memory usage", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 8, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "sort_desc(sum by (pod_name) (rate (container_network_receive_bytes_total{name!=\"\",namespace=~\"^$namespace$\"}[5m])))", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "network", - "refId": "A", - "step": 10 - } - ], - "title": "Pod Network RX", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 13, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.3.4", - "datasource": "${DS_PROMETHEUS}", - "targets": [ - { - "exemplar": true, - "expr": "sort_desc(sum by (pod_name) (rate (container_network_transmit_bytes_total{name!=\"\",namespace=~\"^$namespace$\",kubernetes_io_hostname=~\"$instance\"}[5m])))", - "format": "time_series", - "interval": "10s", - "intervalFactor": 1, - "legendFormat": "{{ pod_name }}", - "metric": "network", - "refId": "B", - "step": 10 - } - ], - "title": "Pod Network TX", - "type": "timeseries" - } - ], - "refresh": "1m", - "schemaVersion": 34, - "style": "dark", - "tags": [ - "kubernetes", - "AKS", - "monitoring" - ], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "description": null, - "error": null, - "hide": 2, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(namespace)", - "description": "Filter by namespace", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "namespace", - "label": "Namespace", - "options": [], - "query": { - "query": "label_values(namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "allValue": ".*", - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(label_kubernetes_io_hostname)", - "description": "Filter by aks node instance", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "instance", - "label": "Instance", - "options": [], - "query": { - "query": "label_values(label_kubernetes_io_hostname)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Kubernetes monitoring by namespace and instance", - "uid": "9AsCXiYn1", - "version": 21, - "weekStart": "monday" - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/llmd-vllm-wva.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/llmd-vllm-wva.yaml deleted file mode 100644 index 886aa83e9..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/llmd-vllm-wva.yaml +++ /dev/null @@ -1,6471 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - annotations: - name: LLM-D - vLLM - WVA - Overview - originalUrl: https://github.com/aleskandro/llm-d-lab/blob/main/manifests/30-grafana/dashboards/llmd-vllm-wva.json - name: llm-d-lab-vllm-wva-overview -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 16, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 999, - "panels": [], - "title": "Library Panels (Imported)", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 40, - "panels": [], - "title": "CPU/Mem (all involved GPU nodes)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*idle/" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 18 - }, - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*iowait/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*irq/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*nice/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*softirq/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*steal/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*system/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*user/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 29, - "maxPerRow": 12, - "options": { - "barRadius": 0, - "barWidth": 0.55, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "orientation": "vertical", - "showValue": "never", - "stacking": "normal", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - }, - "xField": "Time", - "xTickLabelRotation": 30, - "xTickLabelSpacing": 200 - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(sum by (mode) (\n irate(node_cpu_seconds_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])\n) /\n ignoring(mode)\n group_left\nsum (\n irate(node_cpu_seconds_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])\n)\n)", - "format": "time_series", - "interval": "$stacked_charts_min_step", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU (overall)", - "transformations": [ - { - "id": "seriesToRows", - "options": {} - }, - { - "id": "groupingToMatrix", - "options": { - "columnField": "Metric", - "rowField": "Time", - "valueField": "Value" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "steal": true - }, - "includeByName": {}, - "indexByName": { - "Time\\Metric": 0, - "idle": 8, - "iowait": 3, - "irq": 5, - "nice": 7, - "softirq": 4, - "steal": 6, - "system": 1, - "user": 2 - }, - "renameByName": { - "Time\\Metric": "Time", - "idle": "" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Time" - } - ] - } - } - ], - "type": "barchart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "E" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "F" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "G" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "H" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "I" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 33, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "width": 350 - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemTotal_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_MemFree_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Buffers_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Cached_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Slab_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_PageTables_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_SwapCached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (User)", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_PageTables_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (PageTables)", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_SwapCached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (SwapCache)", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Slab_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Slab)", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Cached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Cache)", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Buffers_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Buffers)", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemFree_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Free)", - "range": true, - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(node_memory_SwapTotal_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_SwapFree_bytes{instance=~\"$gpu_node_hostname\"})", - "format": "time_series", - "legendFormat": "{{instance}} (Swap)", - "range": true, - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_HardwareCorrupted_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Corrupted)", - "range": true, - "refId": "I", - "step": 240 - } - ], - "title": "Memory usage (overall)", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 41, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "E" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "F" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "G" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "H" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "I" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 21, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 34, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "width": 350 - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "repeat": "gpu_node_hostname", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemTotal_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_MemFree_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Buffers_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Cached_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_Slab_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_PageTables_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_SwapCached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (User)", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_PageTables_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (PageTables)", - "range": true, - "refId": "B", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_SwapCached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (SwapCache)", - "range": true, - "refId": "C", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Slab_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Slab)", - "range": true, - "refId": "D", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Cached_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Cache)", - "range": true, - "refId": "E", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_Buffers_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Buffers)", - "range": true, - "refId": "F", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_MemFree_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Free)", - "range": true, - "refId": "G", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(node_memory_SwapTotal_bytes{instance=~\"$gpu_node_hostname\"} - node_memory_SwapFree_bytes{instance=~\"$gpu_node_hostname\"})", - "format": "time_series", - "legendFormat": "{{instance}} (Swap)", - "range": true, - "refId": "H", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_memory_HardwareCorrupted_bytes{instance=~\"$gpu_node_hostname\"}", - "format": "time_series", - "legendFormat": "{{instance}} (Corrupted)", - "range": true, - "refId": "I", - "step": 240 - } - ], - "title": "Memory usage ($gpu_node_hostname)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/.*idle/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*iowait/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*irq/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "super-light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*nice/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*softirq/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*steal/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*system/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*user/" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 20, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 28, - "maxPerRow": 12, - "options": { - "barRadius": 0, - "barWidth": 0.55, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "never", - "stacking": "percent", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - }, - "xField": "Time", - "xTickLabelRotation": 75, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.4.0-19923332322", - "repeat": "gpu_node_hostname", - "repeatDirection": "h", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(sum by (instance, mode) (\n irate(node_cpu_seconds_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])\n) /ignoring(mode) group_left sum by (instance) (\n irate(node_cpu_seconds_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])\n) )", - "format": "time_series", - "interval": "$stacked_charts_min_step", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU ($gpu_node_hostname)", - "transformations": [ - { - "id": "labelsToFields", - "options": { - "mode": "columns", - "valueLabel": "mode" - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "partitionByValues", - "options": { - "fields": [ - "instance" - ], - "keepFields": false, - "naming": { - "asLabels": false - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "nice": true, - "steal": true - }, - "includeByName": {}, - "indexByName": { - "Time": 0, - "idle": 8, - "iowait": 3, - "irq": 4, - "nice": 6, - "softirq": 5, - "steal": 7, - "system": 1, - "user": 2 - }, - "renameByName": {} - } - } - ], - "type": "barchart" - } - ], - "title": "CPU/Mem (per node)", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 30, - "panels": [], - "title": "CPU/Mem (per vllm replica)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 45, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=~\"$namespace\", pod=~\"$variant_name.*\", container=\"vllm\"}", - "instant": false, - "legendFormat": "{{node}}", - "range": true, - "refId": "B" - } - ], - "title": "CPU usage per replica", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 47, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod, node) (\n irate(container_cpu_usage_seconds_total{\n namespace=~\"$namespace\",\n container=\"vllm\",\n pod=~\"$variant_name.*\"\n }[$rate_interval])\n)\n/ on (node) group_left\nmachine_cpu_cores", - "hide": true, - "instant": false, - "legendFormat": "{{node}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=~\"$namespace\", pod=~\"$variant_name.*\", container=\"vllm\"}\n/ on (node) group_left\nmachine_cpu_cores", - "instant": false, - "legendFormat": "{{node}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU usage per replica (relative to host node capacity)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 48, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod, container, node, namespace) (\n container_memory_usage_bytes{namespace=~\"$namespace\", container=\"vllm\", pod=~\"$variant_name.*\"}\n)", - "legendFormat": "{{node}}", - "range": true, - "refId": "A" - } - ], - "title": "Mem usage per replica", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 46, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod, container, node, namespace) (\n container_memory_usage_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"$variant_name.*\"}\n)/ on (node) group_left\nsum by (node) (\n label_replace(\n node_memory_MemTotal_bytes{},\n \"node\", \"$1\", # new label \"node\"\n \"instance\", \"(.*)\" # extract value from label \"instance\"\n )\n)", - "legendFormat": "{{node}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory usage per replica (relative to host node capacity)", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 22, - "panels": [], - "title": "GPU", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "hertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "DCGM_FI_DEV_SM_CLOCK{exported_namespace=~\"$namespace\"}", - "legendFormat": "{{modelName}} ({{device}}@{{Hostname}})", - "range": true, - "refId": "A" - } - ], - "title": "GPU SM Clocks", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decmbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 27, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\"}", - "legendFormat": "{{modelName}} ({{device}}@{{Hostname}})", - "range": true, - "refId": "A" - } - ], - "title": "GPU Framebuffer Mem Util.", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\"}", - "legendFormat": "{{modelName}} ({{device}}@{{Hostname}})", - "range": true, - "refId": "A" - } - ], - "title": "GPU Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 26, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{exported_namespace=~\"$namespace\"}", - "legendFormat": "{{modelName}} ({{device}}@{{Hostname}})", - "range": true, - "refId": "A" - } - ], - "title": "GPU Tensor Core Util.", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 1000, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "DCGM_FI_DEV_POWER_USAGE{exported_namespace=~\"$namespace\"}", - "legendFormat": "{{modelName}} ({{device}}@{{Hostname}})", - "range": true, - "refId": "A" - } - ], - "title": "GPU Power Usage", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 57 - }, - "id": 39, - "panels": [], - "title": "I/O (Net, Disk, ...)", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "out (-) / in (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "bps" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "shades" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "shades" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 35, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(node_network_receive_bytes_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])*8", - "format": "time_series", - "legendFormat": "{{device}}@{{instance}} - Rx in", - "range": true, - "refId": "A", - "step": 240 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "-rate(node_network_transmit_bytes_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])*8", - "format": "time_series", - "legendFormat": "{{device}}@{{instance}} - Tx out", - "range": true, - "refId": "B", - "step": 240 - } - ], - "title": "Network traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "reads (-) / writes (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "shades" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "shades" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 37, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "-irate(node_disk_read_bytes_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{device}}@{{instance}} - Read", - "range": true, - "refId": "A", - "step": 240, - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_written_bytes_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])", - "format": "time_series", - "legendFormat": "{{device}}@{{instance}} - Writes", - "range": true, - "refId": "B", - "step": 240 - } - ], - "title": "Disk Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "reads (-) / writes (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "hertz" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "shades" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "shades" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 36, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "-irate(node_disk_reads_completed_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{device}}@{{instance}} - Read", - "range": true, - "refId": "A", - "step": 240, - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(node_disk_writes_completed_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])", - "format": "time_series", - "legendFormat": "{{device}}@{{instance}} - Writes", - "range": true, - "refId": "B", - "step": 240 - } - ], - "title": "Disk IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "reads (-) / writes (+)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 40, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "A" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "palette-classic" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 38, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$gpu_node_hostname\"}[$rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{device}}@{{instance}}", - "range": true, - "refId": "A", - "step": 240, - "useBackend": false - } - ], - "title": "Disk IOPS", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 31, - "panels": [], - "title": "Workload Variant Autoscaler", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 10, - 10 - ], - "fill": "dash" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"}", - "legendFormat": "Current Replicas ({{variant_name}} on {{accelerator_type}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_desired_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"}", - "instant": false, - "legendFormat": "Desired Replicas ({{variant_name}} on {{accelerator_type}})", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "kube_horizontalpodautoscaler_spec_max_replicas{namespace=~\"$namespace\"}", - "instant": false, - "legendFormat": "HPA Max Replicas ({{horizontalpodautoscaler}})", - "range": true, - "refId": "C" - } - ], - "title": "Replicas: Current vs Desired", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "mode": "thresholds" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1.01 - }, - { - "color": "orange", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 42, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{variant_name=~\"$variant_name\",accelerator_type=~\"$accelerator_type\"}", - "legendFormat": "Current Replicas ({{variant_name}} on {{accelerator_type}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_desired_ratio{variant_name=~\"$variant_name\",accelerator_type=~\"$accelerator_type\"}", - "instant": false, - "legendFormat": "Desired Ratio ({{variant_name}} on {{accelerator_type}})", - "range": true, - "refId": "B" - } - ], - "title": "Current vs Desired", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 18, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 43, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "P50", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.1, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "E" - } - ], - "title": "WVA Reconciliation latency", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "hertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 88 - }, - "id": 44, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "orientation": "auto", - "showValue": "never", - "stacking": "normal", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - }, - "xTickLabelRotation": 75, - "xTickLabelSpacing": 0 - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(controller_runtime_reconcile_total{controller=\"variantAutoscaling\"}[$rate_interval])", - "interval": "$stacked_charts_min_step", - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(controller_runtime_reconcile_errors_total{controller=\"variantAutoscaling\"}[$rate_interval])", - "instant": false, - "interval": "$stacked_charts_min_step", - "legendFormat": "error", - "range": true, - "refId": "B" - } - ], - "title": "WVA Reconciliation results", - "transformations": [ - { - "disabled": true, - "id": "seriesToRows", - "options": {} - }, - { - "disabled": true, - "id": "groupingToMatrix", - "options": { - "columnField": "Metric", - "rowField": "Time", - "valueField": "Value" - } - } - ], - "type": "barchart" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 101 - }, - "id": 32, - "panels": [], - "title": "vLLM", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 26, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 102 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(kube_pod_status_ready{namespace=~\"$namespace\"} == 1 and\n on (namespace, pod)\n kube_pod_labels{\n namespace=~\"$namespace\",label_tekton_dev_pipeline_task=\"load-test\", label_benchmark_running=\"true\"\n }\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "A" - } - ], - "title": "Load Generator SIgnal", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 26, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 102 - }, - "id": 1003, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count(kube_pod_status_ready{namespace=~\"$namespace\"} == 1 and\n on (namespace, pod)\n kube_pod_labels{\n namespace=~\"$namespace\",label_tekton_dev_pipeline_task=\"load-test\", label_benchmark_running=\"true\"\n }\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "A" - } - ], - "title": "Load Generator Signal", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 110 - }, - "id": 1002, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.10, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "instant": false, - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "A" - } - ], - "title": "E2E Request Latency ", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 110 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Avg. E2E Latency per instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "G" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "thresholds" - } - }, - { - "id": "custom.pointSize", - "value": 15 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*(y2).*/" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.drawStyle", - "value": "points" - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "H" - }, - "properties": [ - { - "id": "custom.pointSize", - "value": 10 - }, - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 118 - }, - "id": 1001, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.10, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:inter_token_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:inter_token_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"} and (delta(inferno_current_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"}[30s]) > 0)", - "instant": false, - "legendFormat": "Current Replicas - Scale out (y2)", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"} and (delta(inferno_current_replicas{variant_name=\"$variant_name\", accelerator_type=~\"$accelerator_type\"}[30s]) < 0)", - "instant": false, - "legendFormat": "Current Replicas - Scale in (y2)", - "range": true, - "refId": "H" - } - ], - "title": "ITL", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 118 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(vllm:inter_token_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:inter_token_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Avg. ITL per instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 126 - }, - "id": 8, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.1, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "F" - } - ], - "title": "TTFT", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 126 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Avg. TTFT per instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 134 - }, - "id": 17, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:request_prefill_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:request_prefill_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_prefill_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:request_prefill_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.10, sum by(le) (rate(vllm:request_prefill_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:request_prefill_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:request_prefill_time_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "F" - } - ], - "title": "Prefill time", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 134 - }, - "id": 18, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:request_decode_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:request_decode_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_decode_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:request_decode_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.10, sum by(le) (rate(vllm:request_decode_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:request_decode_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:request_decode_time_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "F" - } - ], - "title": "Decode time", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 142 - }, - "id": 14, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:request_queue_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:request_queue_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_queue_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "", - "legendFormat": "P50", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:request_queue_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.1, sum by(le) (rate(vllm:request_queue_time_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:request_queue_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:request_queue_time_seconds_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "F" - } - ], - "title": "Queue Time", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 142 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(vllm:request_queue_time_seconds_sum{model_name=\"$model_name\", namespace=~\"$namespace\"}[$rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Queue Time per instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 150 - }, - "id": 16, - "options": { - "candleStyle": "candles", - "colorStrategy": "close-close", - "colors": { - "down": "red", - "up": "green" - }, - "fields": { - "close": "P75", - "high": "P90", - "low": "P10", - "open": "P25" - }, - "includeAllFields": true, - "legend": { - "calcs": [ - "min", - "max", - "mean", - "p50", - "p90", - "stdDev", - "delta" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "mode": "candles+volume", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:request_max_num_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P90", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(vllm:request_max_num_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P75", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_max_num_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "legendFormat": "P50", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.25, sum by(le) (rate(vllm:request_max_num_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P25", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.1, sum by(le) (rate(vllm:request_max_num_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "interval": "$candle_stick_min_step", - "legendFormat": "P10", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(model_name) (\n rate(vllm:request_max_num_generation_tokens_sum{model_name=\"$model_name\"}[$rate_interval])\n)\n/\nsum by(model_name) (\n rate(vllm:request_max_num_generation_tokens_count{model_name=\"$model_name\"}[$rate_interval])\n)", - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "F" - } - ], - "title": "Max Generation Token in Sequence Group", - "type": "candlestick" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 150 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(vllm:request_max_num_generation_tokens_sum{model_name=\"$model_name\"}[$rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Max Generation Token in Sequence Group per instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "hertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 158 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(vllm:generation_tokens_total{model_name=\"$model_name\"}[$rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Token Throughput (Generated Tokens/s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "hertz" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 158 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(vllm:prompt_tokens_total{model_name=\"$model_name\"}[$rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Token Throughput (Prompt Tokens/s)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic-by-name" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 166 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vllm:num_requests_running{model_name=\"$model_name\"}", - "legendFormat": "Running ({{instance}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vllm:num_requests_waiting{model_name=\"$model_name\"}", - "instant": false, - "legendFormat": "Waiting ({{instance}})", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(vllm:num_requests_running{model_name=\"$model_name\"})\n", - "legendFormat": "Running (total)", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(vllm:num_requests_waiting{model_name=\"$model_name\"})\n", - "legendFormat": "Waiting (total)", - "range": true, - "refId": "D" - } - ], - "title": "vLLM Scheduler State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": true, - "mappings": [], - "max": 1.05, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 166 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vllm:kv_cache_usage_perc{model_name=\"$model_name\"}", - "legendFormat": "KV Cache Util. ({{instance}})", - "range": true, - "refId": "A" - } - ], - "title": "vLLM KV Cache utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 174 - }, - "id": 19, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Spectral", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "multi", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(le) (increase(vllm:request_prompt_tokens_bucket{model_name=\"$model_name\", namespace=~\"$namespace\"}[$rate_interval]))", - "format": "heatmap", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Request Prompt Length", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 174 - }, - "id": 20, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Spectral", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "multi", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "none" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(le) (increase(vllm:request_generation_tokens_bucket{model_name=\"$model_name\", namespace=~\"$namespace\"}[$rate_interval]))", - "format": "heatmap", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Request Generated Tokens Length", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 27, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 182 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "12.4.0-19923332322", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(finished_reason) (increase(vllm:request_success_total{model_name=\"$model_name\", namespace=~\"$namespace\"}[$rate_interval]))", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Response result by Finish Reason", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 42, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": ".*", - "current": { - "text": [ - "experiment-01" - ], - "value": [ - "experiment-01" - ] - }, - "definition": "label_values(vllm:e2e_request_latency_seconds_bucket,namespace)", - "includeAll": true, - "multi": true, - "name": "namespace", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(vllm:e2e_request_latency_seconds_bucket,namespace)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "meta-llama/Llama-3.1-8B", - "value": "meta-llama/Llama-3.1-8B" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(model_name)", - "description": "", - "name": "model_name", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(model_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "ms-inference-scheduling-llm-d-modelservice-decode", - "value": "ms-inference-scheduling-llm-d-modelservice-decode" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(variant_name)", - "name": "variant_name", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(variant_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "allValue": ".*", - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(accelerator_type)", - "includeAll": true, - "name": "accelerator_type", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(accelerator_type)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "ip-10-0-36-72.us-east-2.compute.internal" - ], - "value": [ - "ip-10-0-36-72.us-east-2.compute.internal" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(DCGM_FI_DEV_GPU_TEMP,Hostname)", - "includeAll": false, - "multi": true, - "name": "gpu_node_hostname", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(DCGM_FI_DEV_GPU_TEMP,Hostname)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "1m", - "value": "1m" - }, - "description": "", - "name": "rate_interval", - "options": [], - "query": "1s, 2s, 4s, 8s, 10s, 20s, 30s, 1m, 5m, 10m, 15m, 20m, 30m, 1h", - "type": "custom" - }, - { - "current": { - "text": "5m", - "value": "5m" - }, - "description": "", - "name": "candle_stick_min_step", - "options": [], - "query": "1s, 2s, 4s, 8s, 10s, 20s, 30s, 1m, 5m, 10m, 15m, 20m, 30m, 1h", - "type": "custom" - }, - { - "current": { - "text": "5m", - "value": "5m" - }, - "description": "", - "name": "stacked_charts_min_step", - "options": [], - "query": "1s, 2s, 4s, 8s, 10s, 20s, 30s, 1m, 5m, 10m, 15m, 20m, 30m, 1h", - "type": "custom" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "vLLM - LLM-D - WVA", - "uid": "fcfd3813-b162-4bf5-9bb4-076788452694", - "version": 3 - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/nvidia.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/nvidia.yaml deleted file mode 100644 index dce3247cd..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/nvidia.yaml +++ /dev/null @@ -1,841 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: nvidia - annotations: - name: "NVIDIA DCGM Exporter Dashboard" - originalUrl: "https://grafana.com/grafana/dashboards/12239/" -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "__inputs": [], - "__requires": [ - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.7.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "This dashboard is to display the metrics from DCGM Exporter on a Kubernetes (1.13+) cluster", - "editable": false, - "gnetId": 12239, - "graphTooltip": 0, - "id": null, - "iteration": 1588401887165, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 18, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{instance=~\"${instance}\", gpu=~\"${gpu}\"}", - "instant": false, - "interval": "", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GPU Temperature", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "celsius", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 14, - "options": { - "fieldOptions": { - "calcs": [ - "mean" - ], - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 83 - }, - { - "color": "red", - "value": 87 - } - ] - }, - "unit": "celsius" - }, - "overrides": [], - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.7.3", - "targets": [ - { - "expr": "avg(DCGM_FI_DEV_GPU_TEMP{instance=~\"${instance}\", gpu=~\"${gpu}\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "GPU Avg. Temp", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 18, - "x": 0, - "y": 8 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.5.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{instance=~\"${instance}\", gpu=~\"${gpu}\"}", - "interval": "", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GPU Power Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "watt", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 8 - }, - "id": 16, - "links": [], - "options": { - "fieldOptions": { - "calcs": [ - "sum" - ], - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 2400, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 1800 - }, - { - "color": "red", - "value": 2200 - } - ] - }, - "unit": "watt" - }, - "overrides": [], - "values": false - }, - "orientation": "horizontal", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.7.3", - "targets": [ - { - "expr": "sum(DCGM_FI_DEV_POWER_USAGE{instance=~\"${instance}\", gpu=~\"${gpu}\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "GPU Power Total", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 2, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_DEV_SM_CLOCK{instance=~\"${instance}\", gpu=~\"${gpu}\"} * 1000000", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GPU SM Clocks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "hertz", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{instance=~\"${instance}\", gpu=~\"${gpu}\"}", - "interval": "", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GPU Utilization", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": "100", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "avg": true, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{instance=~\"${instance}\", gpu=~\"${gpu}\"}", - "interval": "", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GPU Framebuffer Mem Used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decmbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{instance=~\"${instance}\", gpu=~\"${gpu}\"}", - "interval": "", - "legendFormat": "GPU {{gpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Tensor Core Utilization", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": false, - "schemaVersion": 22, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(DCGM_FI_DEV_GPU_TEMP, instance)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": true, - "name": "instance", - "options": [], - "query": "label_values(DCGM_FI_DEV_GPU_TEMP, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(gpu)", - "hide": 0, - "includeAll": true, - "index": -1, - "label": null, - "multi": true, - "name": "gpu", - "options": [], - "query": "label_values(gpu)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "NVIDIA DCGM Exporter Dashboard", - "uid": "Oxed_c6Wz", - "variables": { - "list": [] - }, - "version": 1 - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/vllm.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/vllm.yaml deleted file mode 100644 index 37f106d09..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/vllm.yaml +++ /dev/null @@ -1,1627 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: vllm - annotations: - name: vLLM - originalUrl: "" -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Monitoring vLLM Inference Server", - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "End to end request latency measured in seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P95", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P50", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Average", - "range": true, - "refId": "E" - } - ], - "title": "E2E Request Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of tokens processed per second", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(vllm:prompt_tokens_total{model_name=\"$model_name\"}[$rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Prompt Tokens/Sec", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(vllm:generation_tokens_total{model_name=\"$model_name\"}[$rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Generation Tokens/Sec", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Token Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Inter token latency in seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P95", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P50", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(vllm:inter_token_latency_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:inter_token_latency_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Mean", - "range": true, - "refId": "E" - } - ], - "title": "Inter Token Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of requests in RUNNING, WAITING, and SWAPPED state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "vllm:num_requests_running{model_name=\"$model_name\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Num Running", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "vllm:num_requests_waiting{model_name=\"$model_name\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Num Waiting", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "Scheduler State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "P50, P90, P95, and P99 TTFT latency in seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P95", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "P50", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Average", - "range": true, - "refId": "E" - } - ], - "title": "Time To First Token Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Percentage of used cache blocks by vLLM.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "vllm:kv_cache_usage_perc{model_name=\"$model_name\"}", - "instant": false, - "legendFormat": "GPU Cache Usage", - "range": true, - "refId": "A" - } - ], - "title": "Cache Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Heatmap of request prompt length", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 12, - "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "none" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Spectral", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto", - "value": "Request count" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisLabel": "Prompt Length", - "axisPlacement": "left", - "reverse": false, - "unit": "none" - } - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(le) (increase(vllm:request_prompt_tokens_bucket{model_name=\"$model_name\"}[$rate_interval]))", - "format": "heatmap", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{le}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Request Prompt Length", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Heatmap of request generation length", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 13, - "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "none" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Spectral", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto", - "value": "Request count" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisLabel": "Generation Length", - "axisPlacement": "left", - "reverse": false, - "unit": "none" - } - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(le) (increase(vllm:request_generation_tokens_bucket{model_name=\"$model_name\"}[$rate_interval]))", - "format": "heatmap", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{le}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Request Generation Length", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of finished requests by their finish reason: either an EOS token was generated or the max sequence length was reached.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(finished_reason) (increase(vllm:request_success_total{model_name=\"$model_name\"}[$rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Finish Reason", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "seconds", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(vllm:request_queue_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Queue Time", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(vllm:request_prefill_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Prefill", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(vllm:request_decode_time_seconds_sum{model_name=\"$model_name\"}[$rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Decode", - "range": true, - "refId": "B" - } - ], - "title": "Requests Prefill and Decode Time", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(vllm:request_max_num_generation_tokens_sum{model_name=\"$model_name\"}[$rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Tokens", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Max Generation Token in Sequence Group", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "promtheus" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(model_name)", - "hide": 0, - "includeAll": false, - "label": "model_name", - "multi": false, - "name": "model_name", - "options": [], - "query": { - "query": "label_values(model_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "1m", - "value": "1m" - }, - "label": "Rate Interval", - "name": "rate_interval", - "options": [ - { - "selected": false, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "20s", - "value": "20s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "40s", - "value": "40s" - }, - { - "selected": false, - "text": "50s", - "value": "50s" - }, - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "10s,20s,30s,40s,50s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "type": "interval" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": { }, - "timezone": "", - "title": "vLLM", - "uid": "b281712d-8bff-41ef-9f3f-71ad43c05e9b", - "version": 8, - "weekStart": "" - } diff --git a/projects/llm_d_legacy/testing/grafana/dashboards/workload-variant-autoscaler.yaml b/projects/llm_d_legacy/testing/grafana/dashboards/workload-variant-autoscaler.yaml deleted file mode 100644 index 74bccbb2a..000000000 --- a/projects/llm_d_legacy/testing/grafana/dashboards/workload-variant-autoscaler.yaml +++ /dev/null @@ -1,1193 +0,0 @@ -apiVersion: grafana.integreatly.org/v1beta1 -kind: GrafanaDashboard -metadata: - name: workload-variant-autoscaler - annotations: - name: Workload Variant Autoscaler -spec: - allowCrossNamespaceImport: false - instanceSelector: - matchLabels: - dashboards: grafana - resyncPeriod: 10m0s - json: |- - { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Monitoring Workload Variant Autoscaler", - "editable": false, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current vs Desired number of replicas for each variant", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"}", - "instant": false, - "legendFormat": "Current - {{variant_name}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "inferno_desired_replicas{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"}", - "hide": false, - "instant": false, - "legendFormat": "Desired - {{variant_name}}", - "range": true, - "refId": "B" - } - ], - "title": "Replicas: Current vs Desired", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ratio of desired to current replicas (values > 1 indicate scale up needed, < 1 indicate scale down)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": true, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["last"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "inferno_desired_ratio{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"}", - "instant": false, - "legendFormat": "{{variant_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Desired Ratio", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Absolute difference between desired and current replicas", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["last", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "abs(inferno_desired_replicas{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"} - inferno_current_replicas{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"})", - "instant": false, - "legendFormat": "{{variant_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Replica Count Mismatch", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of scaling operations (scale up vs scale down)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["sum"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(inferno_replica_scaling_total{namespace=\"$namespace\",variant_name=~\"$variantName\",direction=\"up\"}[$__rate_interval])", - "instant": false, - "legendFormat": "Scale Up - {{variant_name}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(inferno_replica_scaling_total{namespace=\"$namespace\",variant_name=~\"$variantName\",direction=\"down\"}[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Scale Down - {{variant_name}}", - "range": true, - "refId": "B" - } - ], - "title": "Scaling Operations Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Rate of scaling operations broken down by reason", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["sum"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(reason) (rate(inferno_replica_scaling_total{namespace=\"$namespace\",variant_name=~\"$variantName\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "{{reason}}", - "range": true, - "refId": "A" - } - ], - "title": "Scaling Frequency by Reason", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current number of replicas by variant and accelerator type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "values": false, - "calcs": [ - "lastNotNull" - ], - "fields": "" - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "inferno_current_replicas{namespace=\"$namespace\",variant_name=~\"$variantName\",accelerator_type=~\"$accelerator_type\"}", - "instant": false, - "legendFormat": "{{variant_name}} ({{accelerator_type}})", - "range": true, - "refId": "A" - } - ], - "title": "Current Replicas by Variant", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total scaling operations over time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "values": false, - "calcs": [ - "sum" - ], - "fields": "" - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(increase(inferno_replica_scaling_total{namespace=\"$namespace\",variant_name=~\"$variantName\"}[$__range]))", - "instant": false, - "legendFormat": "Total Scaling Operations", - "range": true, - "refId": "A" - } - ], - "title": "Total Scaling Operations", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Reconciliation latency for the variant autoscaling controller", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "P99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "P95", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"variantAutoscaling\"}[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "P50", - "range": true, - "refId": "C" - } - ], - "title": "Controller Reconciliation Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Controller reconciliation errors and results", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 9, - "options": { - "legend": { - "calcs": ["sum"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(controller_runtime_reconcile_total{controller=\"variantAutoscaling\"}[$__rate_interval])", - "instant": false, - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(controller_runtime_reconcile_errors_total{controller=\"variantAutoscaling\"}[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "errors", - "range": true, - "refId": "B" - } - ], - "title": "Controller Reconciliation Results", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Work queue depth and operations for the variant autoscaling controller", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 10, - "options": { - "legend": { - "calcs": ["last", "max"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "workqueue_depth{controller=\"variantAutoscaling\"}", - "instant": false, - "legendFormat": "Queue Depth", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(workqueue_adds_total{controller=\"variantAutoscaling\"}[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Adds Rate", - "range": true, - "refId": "B" - } - ], - "title": "Work Queue Metrics", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": ["autoscaling", "workload", "variant"], - "templating": { - "list": [ - { - "current": { - "text": "prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(inferno_current_replicas, namespace)", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": { - "query": "label_values(inferno_current_replicas, namespace)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, variant_name)", - "hide": 0, - "includeAll": true, - "label": "variantName", - "multi": true, - "name": "variantName", - "options": [], - "query": { - "query": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, variant_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, accelerator_type)", - "hide": 0, - "includeAll": true, - "label": "accelerator_type", - "multi": true, - "name": "accelerator_type", - "options": [], - "query": { - "query": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, accelerator_type)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, model_name)", - "hide": 0, - "includeAll": true, - "label": "model_name", - "multi": true, - "name": "model_name", - "options": [], - "query": { - "query": "label_values(inferno_current_replicas{namespace=\"$namespace\"}, model_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Workload Variant Autoscaler", - "uid": "wva-dashboard", - "version": 1, - "weekStart": "" - } diff --git a/projects/llm_d_legacy/testing/grafana/datasource.yaml b/projects/llm_d_legacy/testing/grafana/datasource.yaml deleted file mode 100644 index 742fa9f45..000000000 --- a/projects/llm_d_legacy/testing/grafana/datasource.yaml +++ /dev/null @@ -1,31 +0,0 @@ -kind: GrafanaDatasource -apiVersion: grafana.integreatly.org/v1beta1 -metadata: - name: thanos -spec: - resyncPeriod: 10m - uid: prometheus - valuesFrom: - - targetPath: secureJsonData.httpHeaderValue1 - valueFrom: - secretKeyRef: - name: grafana-token - key: token - datasource: - access: proxy - isDefault: true - jsonData: - httpHeaderName1: Authorization - timeInterval: 5s - tlsSkipVerify: true # TODO: don't skip tls verification - secureJsonData: - httpHeaderValue1: "Bearer ${token}" - name: prometheus - type: prometheus - url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - instanceSelector: - matchLabels: - dashboards: grafana - plugins: - - name: grafana-clock-panel - version: 1.3.0 diff --git a/projects/llm_d_legacy/testing/llmisvcs/llmisvc-pd.yaml b/projects/llm_d_legacy/testing/llmisvcs/llmisvc-pd.yaml deleted file mode 100644 index 018d371bb..000000000 --- a/projects/llm_d_legacy/testing/llmisvcs/llmisvc-pd.yaml +++ /dev/null @@ -1,161 +0,0 @@ -# https://kserve.github.io/website/docs/model-serving/generative-inference/llmisvc/llmisvc-configuration#prefill-decode-separation-configuration - -apiVersion: serving.kserve.io/v1alpha1 -kind: LLMInferenceService -metadata: - name: SET_AT_RUNTIME - namespace: SET_AT_RUNTIME - annotations: - security.opendatahub.io/enable-auth: 'false' - prometheus.io/path: /metrics - prometheus.io/port: "8000" -spec: - replicas: 1 - model: - uri: pvc://storage-rwx/gpt-oss-120 - name: openai/gpt-oss-120b - router: - scheduler: - template: - containers: - - name: main - # image: 'ghcr.io/llm-d/llm-d-inference-scheduler:v0.6.0' - command: - - /app/epp - - --pool-name - - '{{ ChildName .ObjectMeta.Name `-inference-pool` }}' - - --pool-namespace - - '{{ .ObjectMeta.Namespace }}' - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --kv-cache-usage-percentage-metric - - vllm:kv_cache_usage_perc - - --enable-cert-reload=true - - --secure-serving=true - - --model-server-metrics-scheme=https - - --cert-path=/var/run/kserve/tls - args: - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: single-profile-handler - - type: queue-scorer - - type: prefix-cache-scorer - - type: max-score-picker - schedulingProfiles: - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: max-score-picker - - nodeSelector: - nvidia.com/gpu.present: "true" - route: {} - gateway: {} - template: - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: >- - --block-size 128 --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - # - name: UCX_LOG_LEVEL - # value: debug - # - name: UCX_PROTO_INFO - # value: "1" - resources: - limits: - nvidia.com/gpu: "1" - rdma/ib: "1" - requests: - rdma/ib: "1" - cpu: '4' - memory: 8Gi - nvidia.com/gpu: "1" - livenessProbe: - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - failureThreshold: 1000 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 - # Prefill workload (separate pool) - prefill: - replicas: 2 - template: - containers: - - name: main - env: - - name: VLLM_ADDITIONAL_ARGS - value: >- - --disable-uvicorn-access-log - --block-size 128 - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' - - name: VLLM_NIXL_SIDE_CHANNEL_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - # - name: UCX_LOG_LEVEL - # value: debug - # - name: UCX_PROTO_INFO - # value: "1" - resources: - limits: - nvidia.com/gpu: "1" - rdma/ib: "1" - requests: - rdma/ib: "1" - cpu: '4' - memory: 64Gi - nvidia.com/gpu: "1" - livenessProbe: - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - failureThreshold: 1000 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 diff --git a/projects/llm_d_legacy/testing/llmisvcs/llmisvc-simple.yaml b/projects/llm_d_legacy/testing/llmisvcs/llmisvc-simple.yaml deleted file mode 100644 index 6febfbc03..000000000 --- a/projects/llm_d_legacy/testing/llmisvcs/llmisvc-simple.yaml +++ /dev/null @@ -1,93 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: LLMInferenceService -metadata: - name: SET_AT_RUNTIME - namespace: SET_AT_RUNTIME - annotations: - security.opendatahub.io/enable-auth: 'false' - prometheus.io/path: /metrics - prometheus.io/port: "8000" -spec: - replicas: 1 - model: - uri: oci://registry.redhat.io/rhelai1/modelcar-llama-3-1-8b-instruct-fp8-dynamic:1.5 - name: llama-3-1-8b-instruct-fp8 - router: - scheduler: - template: - containers: - - name: main - command: - - /app/epp - - --pool-name - - '{{ ChildName .ObjectMeta.Name `-inference-pool` }}' - - --pool-namespace - - '{{ .ObjectMeta.Namespace }}' - - --pool-group - - inference.networking.x-k8s.io - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --cert-path - - /var/run/kserve/tls - args: - - --config-text - - |2 - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: single-profile-handler - - type: prefix-cache-scorer - - type: load-aware-scorer - - type: max-score-picker - schedulingProfiles: - - name: default - plugins: - - pluginRef: prefix-cache-scorer - weight: 2.0 - - pluginRef: load-aware-scorer - weight: 1.0 - - pluginRef: max-score-picker - nodeSelector: - nvidia.com/gpu.present: "true" - route: {} - gateway: {} - template: - tolerations: - - key: "nvidia.com/gpu" - operator: "Exists" - effect: "NoSchedule" - containers: - - name: main - resources: - limits: - nvidia.com/gpu: "1" - requests: - cpu: '4' - memory: 8Gi - nvidia.com/gpu: "1" - livenessProbe: - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 900 - periodSeconds: 60 - timeoutSeconds: 60 - failureThreshold: 1000 - readinessProbe: - failureThreshold: 10000 - httpGet: - path: /health - port: 8000 - scheme: HTTPS - initialDelaySeconds: 60 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 30 diff --git a/projects/llm_d_legacy/testing/prepare_llmd.py b/projects/llm_d_legacy/testing/prepare_llmd.py deleted file mode 100644 index be532825b..000000000 --- a/projects/llm_d_legacy/testing/prepare_llmd.py +++ /dev/null @@ -1,804 +0,0 @@ -#!/usr/bin/env python - -import pathlib -import logging -import datetime -import time -import uuid -import os -import json - -import yaml -import tempfile - -from projects.legacy.library import env, config, run - -TESTING_THIS_DIR = pathlib.Path(__file__).absolute().parent -PSAP_ODS_SECRET_PATH = pathlib.Path( - os.environ.get("PSAP_ODS_SECRET_PATH", "/env/PSAP_ODS_SECRET_PATH/not_set") -) - - -def update_rhoai_pull_secret(secret_file_path): - """ - Updates the cluster pull secret with RHOAI registry credentials - """ - - # Read secret from file - secret = secret_file_path.read_text().strip() - if not secret: - raise ValueError(f"Secret file {secret_file_path} is empty") - - logging.info("Checking current pull secret for quay.io/rhoai registry") - - # Get current pull secret - result = run.run( - 'oc get secret/pull-secret -n openshift-config --template="{{index .data \\".dockerconfigjson\\" | base64decode}}"', - capture_stdout=True, - ) - current_secret = result.stdout - - # Check if registry already configured - if "quay.io/rhoai" in current_secret: - logging.info("Registry quay.io/rhoai already configured in pull secret") - # return - - logging.info("Adding quay.io/rhoai registry to pull secret") - - # Create temporary file with current secret - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: - temp_file.write(current_secret) - temp_file_path = temp_file.name - - try: - # Add registry authentication - run.run( - f'oc registry login --registry=quay.io/rhoai --auth-basic="{secret}" --to="{temp_file_path}"', - log_command=False, - ) - - # Update cluster secret - run.run( - f'oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson="{temp_file_path}"', - log_command=False, - ) - - logging.info("Successfully updated pull secret with quay.io/rhoai registry") - finally: - # Clean up temporary file - os.unlink(temp_file_path) - - -def wait_for_pull_secret_ready(registry="quay.io/rhoai", test_image=None, timeout=300): - """ - Wait for the pull secret to be deployed and working across all nodes - """ - logging.info(f"Waiting for pull secret with {registry} to be deployed...") - start_time = time.time() - - # Flags to track completed steps - secret_contains_registry = False - secret_json_valid = False - machine_config_updated = False - registry_access_tested = False - - while time.time() - start_time < timeout: - try: - # Check that the pull secret contains the registry (only if not already verified) - if not secret_contains_registry: - result = run.run( - 'oc get secret/pull-secret -n openshift-config --template="{{index .data \\".dockerconfigjson\\" | base64decode}}"', - capture_stdout=True, - ) - secret_content = result.stdout - - if registry not in secret_content: - logging.info(f"Pull secret does not yet contain {registry}, waiting...") - time.sleep(10) - continue - else: - logging.info(f"✓ Pull secret contains {registry}") - secret_contains_registry = True - - # Verify the secret is properly formatted JSON (only if not already verified) - if not secret_json_valid: - result = run.run( - 'oc get secret/pull-secret -n openshift-config --template="{{index .data \\".dockerconfigjson\\" | base64decode}}"', - capture_stdout=True, - ) - secret_content = result.stdout - - try: - secret_data = json.loads(secret_content) - if "auths" not in secret_data: - logging.warning("Pull secret missing 'auths' section, waiting...") - time.sleep(10) - continue - - if registry not in secret_data["auths"]: - logging.info(f"Registry {registry} not found in auths section, waiting...") - time.sleep(10) - continue - - # Check that auth data exists - auth_data = secret_data["auths"][registry] - if not auth_data.get("auth") and not auth_data.get("username"): - logging.warning( - f"Registry {registry} missing authentication data, waiting..." - ) - time.sleep(10) - continue - - logging.info("✓ Pull secret JSON structure valid") - secret_json_valid = True - - except json.JSONDecodeError: - logging.warning("Pull secret is not valid JSON, waiting...") - time.sleep(10) - continue - - # Check machine config pool status (only if not already updated) - if not machine_config_updated: - logging.info("Checking machine config propagation...") - result = run.run( - "oc get mcp -o jsonpath='{.items[*].status.conditions[?(@.type==\"Updated\")].status}'", - capture_stdout=True, - ) - if "False" in result.stdout: - logging.info("Machine config pools still updating, waiting...") - time.sleep(30) - continue - else: - logging.info("✓ Machine config pools updated") - machine_config_updated = True - - # Verify nodes can access the registry using crictl (only if not already tested) - if test_image and not registry_access_tested: - logging.info("Validating registry access on nodes...") - - # Get a worker node to test registry access - result = run.run( - "oc get nodes -l node-role.kubernetes.io/worker --no-headers -o custom-columns=NAME:.metadata.name | head -1", - capture_stdout=True, - ) - if result.returncode == 0 and result.stdout.strip(): - test_node = result.stdout.strip() - - logging.info(f"Testing pull of {test_image} from node {test_node}") - - test_result = run.run( - f"oc debug node/{test_node} -- chroot /host crictl pull {test_image}", - capture_stdout=True, - capture_stderr=True, - check=False, - ) - if test_result.returncode != 0: - logging.warning(f"Failed to pull test image: {test_result.stderr}") - logging.warning( - "Registry access test failed, but continuing with deployment" - ) - time.sleep(15) - continue - - logging.info("✓ Successfully pulled test image - registry access confirmed") - registry_access_tested = True - # Clean up the pulled image - run.run( - f"oc debug node/{test_node} -- chroot /host crictl rmi {test_image}", - capture_stdout=True, - capture_stderr=True, - check=False, - ) - else: - logging.warning("No worker nodes found for registry access testing") - registry_access_tested = True # Don't retry the test - - # Check if all required steps are complete - required_tests_complete = ( - secret_contains_registry and secret_json_valid and machine_config_updated - ) - optional_test_complete = not test_image or registry_access_tested - - if required_tests_complete and optional_test_complete: - logging.info(f"Pull secret with {registry} is ready and deployed!") - return True - - # If we get here, some steps are still pending - logging.info("Some validation steps still in progress, continuing to wait...") - - except Exception as e: - logging.warning(f"Error checking pull secret status: {e}") - - time.sleep(15) - - # Timeout reached - logging.error(f"Timeout waiting for pull secret with {registry} to be ready") - return False - - -def prepare(): - """ - Prepares the cluster for LLM-D testing - """ - - if config.project.get_config("prepare.skip"): - logging.info("Preparation skipped") - return - - prepare_operators() - prepare_namespace() - prepare_monitoring() - prepare_grafana() - prepare_rhoai() - prepare_gateway() - scale_up() - - model_ref = config.project.get_config("tests.llmd.inference_service.model") - with run.Parallel("prepare_node") as parallel: - parallel.delayed(download_single_model, model_ref) - - if config.project.get_config("prepare.gpu.wait_for_readiness"): - parallel.delayed(wait_for_gpu_readiness) - - if not config.project.get_config("prepare.preload.skip"): - parallel.delayed(preload_llm_model_image) - - -def prepare_operators(): - """ - Prepares operators defined in the configuration - """ - - if config.project.get_config("prepare.operators.skip"): - logging.info("Operators preparation skipped") - return - - operators_list = config.project.get_config("prepare.operators.list") - if not operators_list: - logging.info("No operators to deploy") - return - - logging.info("Preparing operators") - - with run.Parallel("prepare_operators") as parallel: - for operator_config in operators_list: - parallel.delayed(deploy_operator, operator_config) - - -def deploy_operator(operator_config): - """ - Deploys a single operator based on configuration - """ - - name = operator_config.get("name") - catalog = operator_config.get("catalog", "redhat-operators") - operator = operator_config.get("operator") - namespace = operator_config.get("namespace", "all") - deploy_cr = operator_config.get("deploy_cr", False) - enabled = operator_config.get("enabled", True) - - if not enabled: - logging.info(f"Skipping disabled operator: {name}") - return - - if not operator: - logging.error(f"Operator configuration missing 'operator' field: {operator_config}") - return - - logging.info(f"Deploying operator: {name}") - - # Build the arguments dictionary - args_dict = { - "catalog": catalog, - "manifest_name": operator, - "namespace": namespace, - "artifact_dir_suffix": f"_{operator}", - } - - if deploy_cr is not None: - args_dict["deploy_cr"] = True if deploy_cr is True else str(deploy_cr) - - # Add any extra arguments - extra_args = operator_config.get("extra_args", {}) - if extra_args: - args_dict.update(extra_args) - - run.run_toolbox("cluster", "deploy_operator", **args_dict) - logging.info(f"Successfully deployed operator: {name}") - - -def prepare_grafana(): - """ - Prepares Grafana resources (datasources and dashboards) - """ - - if config.project.get_config("prepare.grafana.skip"): - logging.info("Grafana preparation skipped") - return - - logging.info("Preparing Grafana resources") - - # Get Grafana namespace from config - grafana_namespace = config.project.get_config("prepare.grafana.namespace") - - # Create Grafana namespace if it doesn't exist - logging.info(f"Creating Grafana namespace: {grafana_namespace}") - run.run(f'oc new-project "{grafana_namespace}" --skip-config-write >/dev/null', check=False) - - # Deploy Grafana datasources - datasources = config.project.get_config("prepare.grafana.datasources") - for datasource_path in datasources or []: - datasource_file = TESTING_THIS_DIR / datasource_path - if not datasource_file.exists(): - logging.warning(f"Grafana datasource file not found: {datasource_file}") - continue - - logging.info( - f"Deploying Grafana datasource: {datasource_path} to namespace {grafana_namespace}" - ) - run.run(f"oc apply -n {grafana_namespace} -f {datasource_file}") - - # Deploy Grafana dashboards - dashboards_dir_path = config.project.get_config("prepare.grafana.dashboards_dir") - if dashboards_dir_path: - dashboards_dir = TESTING_THIS_DIR / dashboards_dir_path - dashboard_files = list(dashboards_dir.glob("*.yaml")) - logging.info( - f"Deploying {len(dashboard_files)} Grafana dashboard(s) from {dashboards_dir_path} to namespace {grafana_namespace}" - ) - for dashboard_file in dashboard_files: - logging.info(f" - {dashboard_file.name}") - run.run(f"oc apply -n {grafana_namespace} -f {dashboard_file}") - else: - logging.info("No Grafana dashboards directory configured") - - -def prepare_monitoring(): - """ - Prepares cluster monitoring (user workload monitoring) - """ - - if config.project.get_config("prepare.monitoring.skip"): - logging.info("Monitoring preparation skipped") - return - - logging.info("Preparing cluster monitoring") - - # Get namespaces to enable monitoring for - monitored_namespaces = config.project.get_config("prepare.monitoring.namespaces") - for i, ns in enumerate(monitored_namespaces): - # we fetch the *list*, so the references aren't resolved in 'get_config', - # eg: ['@prepare.namespace.name'] - monitored_namespaces[i] = config.project.resolve_reference(ns) - - # Enable user workload monitoring - run.run_toolbox( - "cluster", "enable_userworkload_monitoring", namespaces=monitored_namespaces or [] - ) - - logging.info("Cluster monitoring setup completed successfully") - - -def scale_up(): - """ - Prepares the cluster for GPU operations - scales up unconditionally if configured - """ - - logging.info("Scaling up the cluster") - - # Scale up the cluster if configured - node_instance_type = config.project.get_config("prepare.cluster.nodes.instance_type") - node_count = config.project.get_config("prepare.cluster.nodes.count") - - if node_instance_type and node_count: - logging.info(f"Scaling cluster to {node_count} {node_instance_type} instances") - run.run_toolbox("cluster", "set_scale", instance_type=node_instance_type, scale=node_count) - - -def wait_for_gpu_readiness(): - """ - Waits for GPU nodes and GPU operator stack to be ready - """ - logging.info("Waiting for GPU nodes to be ready...") - run.run_toolbox("nfd", "wait_gpu_nodes") - run.run_toolbox("gpu-operator", "wait_stack_deployed") - logging.info("GPU readiness check completed") - - -def prepare_rhoai(): - """ - Prepares RHOAI (Red Hat OpenShift AI) - """ - - if config.project.get_config("prepare.rhoai.skip"): - logging.info("RHOAI preparation skipped") - return - - logging.info("Preparing RHOAI") - - # Apply pre-release registry token if available - - rhoai_secret_file = PSAP_ODS_SECRET_PATH / config.project.get_config("secrets.rhoai_rc") - if rhoai_secret_file.exists(): - logging.info("Applying pre-release image registry token") - update_rhoai_pull_secret(rhoai_secret_file) - - # Wait for the pull secret to be deployed and working - rhoai_image = config.project.get_config("prepare.rhoai.image") - rhoai_tag = config.project.get_config("prepare.rhoai.tag") - test_image = f"{rhoai_image}:{rhoai_tag}" - - if not wait_for_pull_secret_ready("quay.io/rhoai", test_image=test_image): - raise RuntimeError("Pull secret deployment failed or timed out") - else: - logging.error(f"RHOAI secret file not found: {rhoai_secret_file}") - - # Apply ImageContentSourcePolicy for quay registry - icsp_yaml = """ -apiVersion: operator.openshift.io/v1alpha1 -kind: ImageContentSourcePolicy -metadata: - name: quay-registry -spec: - repositoryDigestMirrors: - - source: registry.redhat.io/rhoai - mirrors: - - quay.io/rhoai -""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: - f.write(icsp_yaml) - temp_file = f.name - - run.run(f"oc apply -f {temp_file}") - - # Deploy RHOAI - rhoai_image = config.project.get_config("prepare.rhoai.image") - rhoai_tag = config.project.get_config("prepare.rhoai.tag") - rhoai_channel = config.project.get_config("prepare.rhoai.channel") - - run.run_toolbox( - "rhods", "deploy_ods", catalog_image=rhoai_image, tag=rhoai_tag, channel=rhoai_channel - ) - - # Enable KServe - enable_components = config.project.get_config("prepare.rhoai.datasciencecluster.enable") - extra_settings = config.project.get_config("prepare.rhoai.datasciencecluster.extra_settings") - - run.run_toolbox( - "rhods", - "update_datasciencecluster", - enable=enable_components, - extra_settings=extra_settings, - ) - - -def prepare_gateway(): - """ - Prepares the Gateway for LLM inference - """ - - if config.project.get_config("prepare.gateway.skip"): - logging.info("Gateway preparation skipped") - return - - logging.info("Preparing Gateway for LLM inference") - - # Get gateway name from config - gateway_name = config.project.get_config("prepare.gateway.name") - - logging.info(f"Deploying gateway: {gateway_name}") - - run.run_toolbox("llmd", "deploy_gateway", name=gateway_name) - - -def prepare_namespace(): - """ - Prepares the namespace for LLM testing - """ - - namespace = config.project.get_config("prepare.namespace.name") - - logging.info(f"Preparing namespace: {namespace}") - - # Create the namespace - run.run(f'oc new-project "{namespace}" --skip-config-write >/dev/null', check=False) - - return namespace - - -def cleanup_cluster(): - """ - Cleans up the cluster - """ - - if config.project.get_config("prepare.cleanup.skip"): - logging.info("Cleanup skipped") - return - - logging.info("Cleaning up LLM-D cluster resources") - - namespace = config.project.get_config("prepare.namespace.name") - - # Delete namespace - try: - run.run(f"oc delete namespace {namespace} --ignore-not-found") - except Exception as e: - logging.warning(f"Failed to delete namespace {namespace}: {e}") - - # Scale down cluster if configured - auto_scale_down = config.project.get_config("prepare.cluster.nodes.auto_scale_down_on_exit") - node_instance_type = config.project.get_config("prepare.cluster.nodes.instance_type") - - if auto_scale_down and node_instance_type: - logging.info(f"Auto scale down enabled, scaling down {node_instance_type} nodes to 0") - run.run_toolbox("cluster", "set_scale", instance_type=node_instance_type, scale="0") - - -def download_models_to_pvc(): - """ - Download configured models to the PVC for use with inference services - """ - logging.info("Starting model download process") - - # Check if PVC prefetch is enabled - pvc_enabled = config.project.get_config( - "prepare.pvc.enabled", True - ) # Default to True for backward compatibility - - if not pvc_enabled: - logging.info( - "PVC prefetch disabled (prepare.pvc.enabled: false) - skipping model downloads" - ) - return - - try: - # Get models configuration - models_config = config.project.get_config("models") - if not models_config: - logging.info("No models configured for download - skipping") - return - - logging.info(f"PVC prefetch enabled - downloading {len(models_config)} model(s) to PVC...") - - # Download models in parallel for efficiency - with run.Parallel("download_models") as parallel: - for model_key, model_config in models_config.items(): - parallel.delayed(download_single_model, model_key) - - logging.info("Model download process completed successfully") - - except Exception as e: - logging.error(f"Failed to download models to PVC: {e}") - raise - - -def download_single_model(model_key): - """ - Download a single model using the storage toolbox - - Args: - model_key: The key identifying the model in the models configuration - """ - try: - # Check if PVC prefetch is enabled - pvc_enabled = config.project.get_config( - "prepare.pvc.enabled", True - ) # Default to True for backward compatibility - - if not pvc_enabled: - logging.info(f"PVC prefetch disabled - skipping download for model '{model_key}'") - return - - logging.info(f"Starting download for model '{model_key}'") - - # Get model configuration - models_config = config.project.get_config("models") - if model_key not in models_config: - raise ValueError(f"Model '{model_key}' not found in models configuration") - - model_config = models_config[model_key] - if "source" not in model_config: - logging.info(f"Model '{model_key}' has no source configured") - return - - source = model_config["source"] - - # Get PVC configuration - pvc_name = config.project.get_config("prepare.pvc.name") - pvc_size = config.project.get_config("prepare.pvc.size") - pvc_access_mode = config.project.get_config("prepare.pvc.access_mode") - pvc_storage_class = config.project.get_config("prepare.pvc.storage_class", None) - namespace = config.project.get_config("prepare.namespace.name") - downloader_image = config.project.get_config("prepare.model_downloader.image") - - # Check if model already exists in PVC - check_result = run.run( - f"oc get pvc -l {model_key}=yes -oname -n {namespace}", capture_stdout=True, check=False - ) - - expected_pvc = f"persistentvolumeclaim/{pvc_name}" - if check_result.returncode == 0 and expected_pvc in check_result.stdout: - logging.info( - f"Model '{model_key}' already exists in PVC (found {expected_pvc} with label {model_key}=yes) - skipping download" - ) - return - - logging.info( - f"Model '{model_key}' not found in PVC - proceeding with download from {source}" - ) - - # Get credentials path - hf_token_secret = config.project.get_config("secrets.hf_token") - secret_dir = config.project.get_config("secrets.dir.env_key") - hf_creds_path = pathlib.Path(os.environ[secret_dir]) / hf_token_secret - - # Prepare download_to_pvc arguments - download_args = { - "name": model_key, - "source": source, - "pvc_name": pvc_name, - "namespace": namespace, - "pvc_size": pvc_size, - "pvc_access_mode": pvc_access_mode, - "image": downloader_image, - "creds": str(hf_creds_path), - "clean_first": False, # Don't clean to allow multiple models in same PVC - "run_as_root": config.project.get_config("security.run_as_root"), - } - - # Add storage class if configured - if pvc_storage_class: - download_args["pvc_storage_class_name"] = pvc_storage_class - logging.info(f"Using PVC storage class: {pvc_storage_class}") - - run.run_toolbox("llmd", "download_to_pvc", **download_args) - - logging.info(f"Successfully downloaded model '{model_key}'") - - except Exception as e: - logging.error(f"Failed to download model '{model_key}': {e}") - raise - - -def preload_llm_model_image(): - """ - Preloads the LLM model image on GPU nodes to reduce startup time - """ - - logging.info("Preloading LLM model image on GPU nodes") - - try: - all_images = {} - - # Get the model image URI from the YAML file - llmisvc_file = config.project.get_config("tests.llmd.inference_service.yaml_file") - if not llmisvc_file: - return - - yaml_file = TESTING_THIS_DIR / "llmisvcs" / llmisvc_file - - namespace = config.project.get_config("tests.llmd.namespace") - - # Extract the model URI using yq - image_result = run.run(f"cat {yaml_file} | yq .spec.model.uri", capture_stdout=True) - - if image_result.returncode != 0: - raise RuntimeError( - f"Failed to extract model URI from {yaml_file}: {image_result.stderr}" - ) - - model_uri = image_result.stdout.strip().strip('"') - if model_uri.startswith("oci://"): - model_image = model_uri.strip("oci://") - image_name = ( - run.run(f"cat {yaml_file} | yq .spec.model.name", capture_stdout=True) - .stdout.strip() - .strip('"') - ) - logging.info(f"Preloading model image: {model_image} ({image_name})") - all_images[image_name] = model_image - else: - logging.info(f"Model URI ({model_uri}) isn't an OCI image, ignoring.") - - # Get additional images from RHODS operator CSV - logging.info("Fetching additional images from RHODS operator CSV") - - # First get the actual CSV name - csv_name_result = run.run( - "oc get csv -n redhat-ods-operator -l operators.coreos.com/rhods-operator.redhat-ods-operator -oname", - capture_stdout=True, - check=False, - ) - - if csv_name_result.returncode != 0: - logging.warning(f"Failed to get RHODS operator CSV name: {csv_name_result.stderr}") - additional_images = [] - else: - csv_name = csv_name_result.stdout.strip().replace( - "clusterserviceversion.operators.coreos.com/", "" - ) - logging.info(f"Found RHODS operator CSV: {csv_name}") - - csv_result = run.run( - f"oc get csv {csv_name} -n redhat-ods-operator -ojson | jq '.spec.relatedImages'", - capture_stdout=True, - ) - - if csv_result.returncode != 0: - logging.warning( - f"Failed to get RHODS operator related images (return code: {csv_result.returncode}): {csv_result.stderr}" - ) - additional_images = [] - else: - # Parse the JSON output to get specific images - related_images = json.loads(csv_result.stdout) - - # Extract the specific images we need - target_image_names = [ - "rhaiis_vllm_cuda_image", - "odh_llm_d_inference_scheduler_image", - "odh_llm_d_routing_sidecar_image", - ] - - logging.info(f"Parsed {len(related_images)} related images from CSV") - - additional_images = [] - - for i, img in enumerate(related_images): - img_name = img["name"] - if img_name not in target_image_names: - continue - all_images[img_name] = img["image"] - logging.info(f"Found additional image to preload: {img_name} = {img['image']}") - - logging.info( - f"Found {len(additional_images)} additional images to preload out of {len(target_image_names)} targets" - ) - - # Get extra images from config file - extra_images = config.project.get_config("prepare.preload.extra_images", {}) - if extra_images: - logging.info(f"Adding {len(extra_images)} extra images from config") - for image_name, image_uri in extra_images.items(): - all_images[image_name] = image_uri - logging.info(f"Added extra image to preload: {image_name} = {image_uri}") - else: - logging.info("No extra images configured for preloading") - - # Preload all images in parallel - logging.info(f"Starting parallel preload of {len(all_images)} images") - - with run.Parallel("preload_images") as parallel: - for image_name, image in all_images.items(): - parallel.delayed(preload_single_image, namespace, image, image_name) - - logging.info("LLM model and related images preloading completed successfully") - - except Exception as e: - logging.error(f"Failed to preload LLM model image: {e}") - raise - - -def preload_single_image(namespace, image, image_name=""): - """ - Preloads a single image on GPU nodes - """ - try: - logging.info(f"Preloading image{f' {image_name}' if image_name else ''}: {image}") - node_selector_key = config.project.get_config("prepare.preload.node_selector_key") - node_selector_value = config.project.get_config("prepare.preload.node_selector_value") - - run.run_toolbox( - "cluster", - "preload_image", - name=image_name or "preload", - namespace=namespace, - node_selector_key=node_selector_key, - node_selector_value=node_selector_value, - image=image, - ) - except Exception as e: - logging.error( - f"Failed to preload image{f' {image_name}' if image_name else ''} {image}: {e}" - ) - raise diff --git a/projects/llm_d_legacy/testing/presets.d/presets.yaml b/projects/llm_d_legacy/testing/presets.d/presets.yaml deleted file mode 100644 index 984e6126e..000000000 --- a/projects/llm_d_legacy/testing/presets.d/presets.yaml +++ /dev/null @@ -1,105 +0,0 @@ -__multiple: true - -# Preset that automatically scales up GPU nodes if none exist, and scales down at the end -scale_up: - prepare.cluster.nodes.auto_scale: true - prepare.cluster.nodes.auto_scale_down_on_exit: true - -dev: - matbench.enabled: false - tests.llmd.inference_service.model: facebook-opt-125m - -opt-125m: - tests.llmd.inference_service.model: facebook-opt-125m - tests.llmd.inference_service.vllm_args[5]: "--max-model-len=2048" - -psap_h200: - extends: [llama-70b] - tests.capture_prom: false - tests.capture_prom_uwm: false - tests.llmd.skip_prepare: true - prepare.namespace.name: kpouget-dev - prepare.cluster.skip: true - tests.llmd.inference_service.gateway.name: gateway-internal - -llmisvc_pd: - tests.llmd.flavors: [pd-x2-ptp1-px4-dtp4] - -pvc_rwx: - prepare.pvc.name: storage-rwx - prepare.pvc.access_mode: ReadWriteMany - -azure: - security.run_as_root: true - prepare.preload.skip: true - prepare.operators.skip: true - prepare.cluster.skip: true - prepare.rhoai.skip: true - - tests.capture_prom: false - tests.capture_prom_uwm: false - tests.llmd.skip_prepare: false - -azure_light: - extends: [azure, opt-125m] - prepare.pvc.storage_class: managed-csi - -cks: - extends: [pvc_rwx, llama-70b] - - tests.capture_prom: false - tests.capture_prom_uwm: false - tests.llmd.inference_service.metrics.manual_capture: false - tests.llmd.inference_service.gateway.name: gateway-internal - tests.llmd.inference_service.image_pull_secrets: kpouget-pull-secret - - tests.llmd.skip_prepare: true - prepare.namespace.name: kpouget-dev - prepare.preload.node_selector_key: gpu.nvidia.com/class - prepare.preload.node_selector_value: "H200" - tests.llmd.inference_service.extra_properties: - spec.template.affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: NotIn - values: - - gf48e48 - - gf4334a - prepare.preload.extra_images: - vllm-cuda-rhel9: registry.redhat.io/rhaiis/vllm-cuda-rhel9@sha256:094db84a1da5e8a575d0c9eade114fa30f4a2061064a338e3e032f3578f8082a - llm-d-inference-scheduler: ghcr.io/opendatahub-io/rhaii-on-xks/llm-d-inference-scheduler:e6b5db0@sha256:43e8b8edc158f31535c8b23d77629f8cde111cc762a8f4ee5f2f884470566211 - guidellm: ghcr.io/vllm-project/guidellm:v0.5.4 - -baseline-flavors: - tests.llmd.flavors: [simple, simple-tp4, simple-tp2-x4] - -intelligentrouting-flavors: - tests.llmd.flavors: [intelligentrouting-tp2-x4] - -guidellm_light: - tests.llmd.benchmarks.guidellm.load_shape_name: lightweight - tests.llmd.benchmarks.guidellm.rate: "1,10,50" - tests.llmd.benchmarks.guidellm.args.data: prompt_tokens=256,output_tokens=128 - tests.llmd.benchmarks.guidellm.args.max_seconds: 30 - -guidellm_multiturn_eval: - tests.llmd.benchmarks.guidellm.load_shape_name: Multiturn - tests.llmd.benchmarks.guidellm.rate: [32, 64, 128, 256, 512] # keep as a list, multi-rate not supported by guidellm-multiturn - tests.llmd.benchmarks.guidellm.args.data: "prompt_tokens=128,output_tokens=128,turns=5,prefix_tokens=10000,prefix_count={2*rate}" - tests.llmd.benchmarks.guidellm.args.max_requests: "{10*rate}" - -guidellm_heterogeneous_eval: - tests.llmd.benchmarks.guidellm.load_shape_name: Heterogeneous - tests.llmd.benchmarks.guidellm.rate: "1,10,50,100,200,300" - tests.llmd.benchmarks.guidellm.args.data: prompt_tokens=8000,prompt_tokens_stdev=8500,prompt_tokens_min=50,prompt_tokens_max=30000,output_tokens=800,output_tokens_stdev=1500,output_tokens_min=20,output_tokens_max=8000 - tests.llmd.benchmarks.guidellm.args.max_seconds: 600 - -gpt-oss: - tests.llmd.inference_service.model: gpt-oss-120 - tests.llmd.benchmarks.guidellm.args.request_type: text_completions - -llama-70b: - tests.llmd.inference_service.model: llama3-3-70b diff --git a/projects/llm_d_legacy/testing/test.py b/projects/llm_d_legacy/testing/test.py deleted file mode 100755 index a9011b1b8..000000000 --- a/projects/llm_d_legacy/testing/test.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python - -import sys, os -import pathlib -import subprocess -import logging - -logging.getLogger().setLevel(logging.INFO) -import datetime -import time -import functools -import uuid - -import yaml -import fire - -from projects.legacy.library import env, config, run, common -from projects.matrix_benchmarking.library import visualize - -import prepare_llmd, test_llmd - -TESTING_THIS_DIR = pathlib.Path(__file__).absolute().parent - -# PSAP_ODS_SECRET_PATH = pathlib.Path(os.environ.get("PSAP_ODS_SECRET_PATH", "/env/PSAP_ODS_SECRET_PATH/not_set")) - -initialized = False - - -def init(ignore_secret_path=False, apply_preset_from_pr_args=True): - global initialized - if initialized: - logging.debug("Already initialized.") - return - initialized = True - - env.init() - config.init(TESTING_THIS_DIR, apply_preset_from_pr_args) - - -def entrypoint(ignore_secret_path=False, apply_preset_from_pr_args=True): - def decorator(fct): - @functools.wraps(fct) - def wrapper(*args, **kwargs): - init(ignore_secret_path, apply_preset_from_pr_args) - exit_code = fct(*args, **kwargs) - logging.info(f"exit code of {fct.__qualname__}: {exit_code}") - if exit_code is None: - exit_code = 0 - raise SystemExit(exit_code) - - return wrapper - - return decorator - - -# --- - - -@entrypoint() -def prepare_ci(): - """ - Prepares the cluster and the namespace for running the tests - """ - - prepare_llmd.prepare() - - -@entrypoint() -def test_ci(): - """ - Runs the test from the CI - """ - - try: - test_artifact_dir_p = [None] - test_artifact_dir_p[0] = env.ARTIFACT_DIR - - failed = test_llmd.test() - logging.info("test_llmd.test " + ("failed" if failed else "passed")) - - return 1 if failed else 0 - finally: - if config.project.get_config("clusters.cleanup_on_exit"): - cleanup_cluster() - - export.export_artifacts(env.ARTIFACT_DIR, test_step="test_ci") - - -@entrypoint(ignore_secret_path=True, apply_preset_from_pr_args=False) -def generate_plots_from_pr_args(): - """ - Generates the visualization reports from the PR arguments - """ - - visualize.download_and_generate_visualizations() - - export.export_artifacts(env.ARTIFACT_DIR, test_step="plot") - - -@entrypoint() -def cleanup_cluster(mute=False): - """ - Restores the cluster to its original state - """ - # _Not_ executed in OpenShift CI cluster (running on AWS). Only required for running in bare-metal environments. - - common.cleanup_cluster() - - prepare_llmd.cleanup_cluster() - - -@entrypoint(ignore_secret_path=True) -def generate_plots(results_dirname): - visualize.generate_from_dir(str(results_dirname)) - - -@entrypoint(ignore_secret_path=True) -def export_artifacts(artifacts_dirname): - export.export_artifacts(artifacts_dirname) - - -@entrypoint(ignore_secret_path=True) -def prepare_namespace(): - """ - Prepares the namespace where the LLM inference services will be deployed - """ - - return prepare_llmd.prepare_namespace() - - -@entrypoint() -def matbench_run_one(): - """ - Runs one test as part of a MatrixBenchmark benchmark - """ - - test_llmd.matbench_run_one() - - -# --- - - -class Entrypoint: - """ - Commands for launching the CI tests - """ - - def __init__(self): - self.pre_cleanup_ci = cleanup_cluster - self.post_cleanup_ci = cleanup_cluster - self.prepare_ci = prepare_ci - self.test_ci = test_ci - - self.export_artifacts = export_artifacts - - self.generate_plots_from_pr_args = generate_plots_from_pr_args - self.generate_plots = generate_plots - self.prepare_namespace = prepare_namespace - - self.matbench_run_one = matbench_run_one - - -# --- - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - fire.Fire(Entrypoint()) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - logging.error(f"Command '{e.cmd}' failed --> {e.returncode}") - sys.exit(1) - except KeyboardInterrupt: - print() # empty line after ^C - logging.error(f"Interrupted.") - sys.exit(1) diff --git a/projects/llm_d_legacy/testing/test_llmd.py b/projects/llm_d_legacy/testing/test_llmd.py deleted file mode 100644 index e3d6f30d3..000000000 --- a/projects/llm_d_legacy/testing/test_llmd.py +++ /dev/null @@ -1,1502 +0,0 @@ -#!/usr/bin/env python - -import pathlib -import logging -import datetime -import time -import uuid -import os -import json - -import yaml - -from projects.legacy.library import env, config, run - -# from projects.cluster.library import prom -from projects.matrix_benchmarking.library import visualize -import prepare_llmd - -TESTING_THIS_DIR = pathlib.Path(__file__).absolute().parent - - -def test_single_flavor(flavor, flavor_index, total_flavors, namespace): - """ - Run tests for a single flavor - - Args: - flavor: The flavor string to test - flavor_index: Current flavor number (1-indexed) - total_flavors: Total number of flavors being tested - namespace: Kubernetes namespace to use - - Returns: - False if test succeeded, Exception or True if test failed - """ - logging.info(f"Running test for flavor: {flavor} ({flavor_index}/{total_flavors})") - - flavor_failed = False - prom_start_ts = None - - llmisvc_name = config.project.get_config("tests.llmd.inference_service.name") - llmisvc_name += f"-{flavor}" - - with env.NextArtifactDir(f"flavor_{flavor}"): - try: - # Clean up any existing LLM inference services and pods before testing - if not config.project.get_config("tests.llmd.inference_service.skip_deployment"): - cleanup_llm_inference_resources() - - # Reset Prometheus before testing - if config.project.get_config("tests.capture_prom"): - logging.info("Resetting Prometheus database before testing") - prom_start_ts = prom.reset_prometheus() - - # Extract the model name from configuration - model_ref = config.project.get_config("tests.llmd.inference_service.model") - - # Deploy LLM inference service - apply_llmisvc_deployment_profile(flavor) - - _, _, llmisvc_path = deploy_llm_inference_service( - flavor, llmisvc_name, namespace, model_ref - ) - - models = config.project.get_config(f"models") - model_name = models[model_ref]["name"] - logging.info(f"Using model: {model_name} (from config reference: {model_ref})") - - # Get the service URL - endpoint_url = get_llm_inference_url(llmisvc_name, namespace, flavor) - - if config.project.get_config("tests.llmd.inference_service.do_simple_test"): - if not test_llm_inference_simple(endpoint_url, llmisvc_name, namespace, model_name): - raise RuntimeError("Simple inference test failed :/") - - # Run benchmarks - if config.project.get_config("tests.llmd.benchmarks.guidellm.enabled"): - flavor_failed |= run_guidellm_benchmark(endpoint_url, llmisvc_name, namespace) - - except Exception as e: - logging.exception(f"Test failed for flavor {flavor} :/") - flavor_failed = e - - finally: - # Always capture LLM inference service state (success or failure) - logging.info("Capturing LLM inference service state for debugging") - try: - capture_llm_inference_service_state(llmisvc_name, namespace) - except Exception as capture_e: - logging.warning(f"Failed to capture LLM inference service state: {capture_e}") - - # Always dump Prometheus data after testing (success or failure) - logging.info("Dumping Prometheus database after testing") - namespace = config.project.get_config("tests.llmd.namespace") - if prom_start_ts: - prom.dump_prometheus(prom_start_ts, namespace) - - # Clean up LLM inference service resources after this flavor - logging.info(f"Cleaning up LLM inference service resources for flavor {flavor}") - try: - cleanup_llm_inference_resources() - except Exception as cleanup_e: - logging.warning(f"Failed to cleanup LLM inference service resources: {cleanup_e}") - - # Generate test metadata files for this flavor - _generate_test_metadata(flavor_failed, flavor) - - return flavor_failed - - -def test(): - """ - Runs the main LLM-D test - """ - - if config.project.get_config("tests.llmd.skip"): - logging.info("LLM-D test skipped") - return False - - logging.info("Running LLM-D test") - - # Get test flavors from config - flavors = config.project.get_config("tests.llmd.flavors") - if not isinstance(flavors, list): - flavors = [flavors] - - if config.project.get_config("tests.llmd.inference_service.skip_deployment"): - if len(flavors) != 1: - raise ValueError( - "tests.llmd.inference_service.skip_deployment is set " - f"but got multiple flavors to deploy: {', '.join(flavors)}" - ) - - if not config.project.get_config("tests.llmd.skip_prepare"): - prepare_for_test() - else: - logging.info("Skipping test preparation (tests.llmd.skip_prepare=True)") - model_ref = config.project.get_config("tests.llmd.inference_service.model") - prepare_llmd.download_single_model(model_ref) - - logging.info(f"Running tests for flavors: {flavors}") - - overall_failed = False - test_directory = None - namespace = config.project.get_config("tests.llmd.namespace") - - with env.NextArtifactDir("llm_d_testing"): - test_directory = env.ARTIFACT_DIR - - for i, flavor in enumerate(flavors): - flavor_failed = test_single_flavor(flavor, i + 1, len(flavors), namespace) - flavor_failed = False - - if flavor_failed: - overall_failed = True - logging.error(f"Test failed for flavor: {flavor}") - else: - logging.info(f"Test completed successfully for flavor: {flavor}") - - # Final cleanup of all LLM inference service resources after all flavors complete - logging.info("Final cleanup of all LLM inference service resources") - try: - cleanup_llm_inference_resources() - except Exception as cleanup_e: - logging.warning( - f"Failed to perform final cleanup of LLM inference service resources: {cleanup_e}" - ) - - # Handle conditional GPU scaling after all tests complete - conditional_scale_down() - - # Generate visualization if enabled - if not config.project.get_config("tests.visualize"): - logging.info("Visualization disabled.") - else: - exc = generate_visualization(test_directory, overall_failed) - - if exc: - logging.error(f"Test visualization failed: {exc}") - - if exc and not overall_failed: - raise exc - - if overall_failed and isinstance(overall_failed, Exception): - raise overall_failed - - return overall_failed - - -def prepare_for_test(): - # Handle conditional GPU scaling before test - conditional_scale_up() - - # Ensure GPU nodes are available before running test - ensure_gpu_nodes_available() - # Run GPU readiness check and image preloading in parallel - logging.info("Starting parallel GPU readiness check and image/model preloading") - - model_ref = config.project.get_config("tests.llmd.inference_service.model") - - with run.Parallel("prepare_gpu_node") as parallel: - if config.project.get_config("prepare.gpu.wait_for_readiness"): - parallel.delayed(prepare_llmd.wait_for_gpu_readiness) - if not config.project.get_config("prepare.preload.skip"): - parallel.delayed(prepare_llmd.preload_llm_model_image) - - parallel.delayed(prepare_llmd.download_single_model, model_ref) - - -def _generate_test_metadata(failed, flavor): - """ - Generate metadata files for the test execution - """ - logging.info(f"Generating test metadata files for flavor: {flavor}") - - # Write exit code file - exit_code = "1" if failed else "0" - exit_code_path = env.ARTIFACT_DIR / "exit_code" - with open(exit_code_path, "w") as f: - f.write(exit_code) - - logging.info(f"Written exit code: {exit_code} to {exit_code_path}") - - # Write settings file using YAML - settings_path = env.ARTIFACT_DIR / "settings.yaml" - settings_data = {"llm-d": True, "flavor": flavor} - - with open(settings_path, "w") as f: - yaml.dump(settings_data, f, default_flow_style=False) - - logging.info(f"Written settings to {settings_path} for flavor: {flavor}") - - -def generate_visualization(test_artifact_dir, test_failed): - """ - Generate visualization from test artifacts - """ - exc = None - - with env.NextArtifactDir("plots"): - exc = run.run_and_catch( - exc, visualize.generate_from_dir, test_artifact_dir, test_failed=test_failed - ) - if not exc: - logging.info( - f"Test visualization has been generated into {env.ARTIFACT_DIR}/reports_index.html" - ) - else: - logging.info(f"Test failed. See '{env.ARTIFACT_DIR}' for more details. {exc}") - - return exc - - -def apply_llmisvc_deployment_profile(flavor): - """ - Apply deployment profile configuration based on flavor - - Always applies __default__ profile first, then applies the base_flavor profile - - Args: - flavor: The flavor string to extract base flavor from - """ - logging.info(f"Applying deployment profile for flavor: {flavor}") - - # Parse flavor to get the base flavor - components = parse_flavor_components(flavor) - base_flavor = components["base"] - - # Get deployment profiles from config - deployment_profiles = config.project.get_config("deployment_profiles") - - # Always apply __default__ profile first - logging.info("Applying __default__ deployment profile") - for config_key, config_value in deployment_profiles["__default__"].items(): - logging.info(f"Setting {config_key} = {config_value}") - config.project.set_config(config_key, config_value) - - if base_flavor == "__default__": - return - - # Apply base flavor profile - logging.info(f"Applying {base_flavor} deployment profile") - for config_key, config_value in deployment_profiles[base_flavor].items(): - logging.info(f"Setting {config_key} = {config_value}") - config.project.set_config(config_key, config_value) - - -def parse_flavor_components(flavor): - """ - Parse flavor string into components dictionary - - Examples: - - 'intelligentrouting-tp2-x2' -> {'base': 'intelligentrouting', 'tp_size': 2, 'replicas': 2} - - 'simple' -> {'base': 'simple', 'tp_size': None, 'replicas': None} - - 'pd-x2-tp4-ptp1-px4' -> {'base': 'pd', 'tp_size': 4, 'replicas': 2, 'prefill_tp_size': 1, 'prefill_replicas': 4} - - 'pd-x2-dtp4-ptp1-px4' -> {'base': 'pd', 'tp_size': 4, 'replicas': 2, 'prefill_tp_size': 1, 'prefill_replicas': 4} - - Args: - flavor: The flavor string to parse - - Returns: - dict: {'base': str, 'tp_size': int|None, 'replicas': int|None, 'prefill_tp_size': int|None, 'prefill_replicas': int|None} - """ - parts = flavor.split("-") - components = { - "base": parts[0], - "tp_size": None, - "replicas": None, - "prefill_tp_size": None, - "prefill_replicas": None, - } - # Mapping of prefixes to field names and descriptions (order matters - longer prefixes first) - prefix_map = { - "ptp": ("prefill_tp_size", "prefill TP", "ptp (e.g., ptp1, ptp2)"), - "px": ("prefill_replicas", "prefill replica", "px (e.g., px2, px4)"), - "dtp": ("tp_size", "decode TP", "dtp (e.g., dtp2, dtp8)"), - "tp": ("tp_size", "TP", "tp (e.g., tp2, tp8)"), - "x": ("replicas", "replica", "x (e.g., x2, x4)"), - } - - # Parse remaining parts - for part in parts[1:]: - matched = False - for prefix, (field, desc, format_example) in prefix_map.items(): - if not part.startswith(prefix): - continue - try: - components[field] = int(part[len(prefix) :]) - matched = True - break - except ValueError: - raise ValueError( - f"Invalid {desc} specification: {part}. Expected format: {format_example}" - ) - - if not matched: - raise ValueError( - f"Unknown flavor component: {part}. Expected format: -[tp|dtp]-[x]-[ptp]-[px]" - ) - - logging.info(f"Parsed flavor '{flavor}' -> {components}") - return components - - -def apply_flavor_modifications(isvc_data, flavor): - """ - Apply flavor-specific modifications to the ISVC data - - Args: - isvc_data: The loaded YAML data structure - flavor: The flavor string to apply - """ - logging.info(f"Applying flavor modifications for: {flavor}") - - # Parse the flavor components - components = parse_flavor_components(flavor) - base_flavor = components["base"] - tp_size = components["tp_size"] - replicas = components["replicas"] - prefill_tp_size = components["prefill_tp_size"] - prefill_replicas = components["prefill_replicas"] - - if base_flavor in ["pd"]: - if "prefill" not in isvc_data["spec"]: - raise ValueError(f"No spec.prefill field in the {base_flavor} LLMISVC") - else: - if "prefill" in isvc_data["spec"]: - raise ValueError(f"Field spec.prefill unexpected in the {base_flavor} LLMISVC") - - # Apply base flavor modifications - if base_flavor == "simple": - logging.info("Applying 'simple' flavor - keeping ISVC untouched") - # No modifications - keep original ISVC configuration - - elif base_flavor in ["intelligentrouting"]: - logging.info("Applying 'intelligent-routing' flavor - keeping ISVC untouched") - # No modifications - keep original ISVC configuration - elif base_flavor in ["pd"]: - logging.info("Applying 'pd' flavor - keeping ISVC untouched") - # No modifications - keep original ISVC configuration - elif base_flavor == "default": - logging.info("Applying 'default' flavor - keeping ISVC untouched") - # No modifications - keep original ISVC configuration - - else: - # Unknown flavor - raise ValueError( - f"Unknown base flavor '{base_flavor}'. Supported flavors: simple, intelligentrouting" - ) - - # Ensure spec section exists - if "spec" not in isvc_data: - isvc_data["spec"] = {} - - # Apply replica scaling from flavor - if replicas is not None: - isvc_data["spec"]["replicas"] = replicas - logging.info(f"Setting spec.replicas = {replicas}") - - # Apply tensor parallelism from flavor - if tp_size is not None: - apply_flavor_tensor_parallelism(isvc_data, tp_size) - - # Apply prefill-specific configurations for P/D deployments - if base_flavor == "pd": - if prefill_replicas is not None: - # Multiply prefill replicas by main replicas - final_prefill_replicas = prefill_replicas * (replicas if replicas is not None else 1) - isvc_data["spec"]["prefill"]["replicas"] = final_prefill_replicas - logging.info( - f"Setting spec.prefill.replicas = {final_prefill_replicas} (p.x{prefill_replicas} * x{replicas if replicas is not None else 1})" - ) - - if prefill_tp_size is not None: - apply_prefill_tensor_parallelism(isvc_data, prefill_tp_size) - - -def apply_flavor_tensor_parallelism(isvc_data, tp_size): - """ - Apply tensor parallelism configuration from flavor to the ISVC - - Args: - isvc_data: The loaded YAML data structure - tp_size: Tensor parallel size from flavor - """ - logging.info(f"Applying tensor parallelism from flavor: TP={tp_size}") - - # Ensure template.containers section exists - if "template" not in isvc_data["spec"]: - isvc_data["spec"]["template"] = {} - if "containers" not in isvc_data["spec"]["template"]: - isvc_data["spec"]["template"]["containers"] = [{"name": "main"}] - - # Find the main container - main_container = None - for container in isvc_data["spec"]["template"]["containers"]: - if container.get("name") == "main": - main_container = container - break - - if not main_container: - main_container = {"name": "main"} - isvc_data["spec"]["template"]["containers"].append(main_container) - - # Ensure env section exists - if "env" not in main_container: - main_container["env"] = [] - - # Find or create VLLM_ADDITIONAL_ARGS environment variable - vllm_args_env = None - for env_var in main_container["env"]: - if env_var.get("name") == "VLLM_ADDITIONAL_ARGS": - vllm_args_env = env_var - break - - if not vllm_args_env: - vllm_args_env = {"name": "VLLM_ADDITIONAL_ARGS", "value": ""} - main_container["env"].append(vllm_args_env) - - # Add tensor parallelism argument - current_args = vllm_args_env.get("value", "").strip() - tp_arg = f"--tensor-parallel-size={tp_size}" - - # Check if TP argument already exists and update it - import re - - if re.search(r"--tensor-parallel-size=\d+", current_args): - # Replace existing TP argument - current_args = re.sub(r"--tensor-parallel-size=\d+", tp_arg, current_args) - else: - # Add new TP argument - if current_args: - current_args = f"{current_args} {tp_arg}" - else: - current_args = tp_arg - - vllm_args_env["value"] = current_args - - # Set GPU resources to match tensor parallel size - apply_gpu_resources(main_container, tp_size) - - logging.info(f"Applied flavor TP: {tp_arg}, GPU resources: {tp_size}") - - -def apply_prefill_tensor_parallelism(isvc_data, tp_size): - """ - Apply tensor parallelism configuration to the prefill container - - Args: - isvc_data: The loaded YAML data structure - tp_size: Tensor parallel size for prefill - """ - logging.info(f"Applying prefill tensor parallelism: TP={tp_size}") - - # Find the main container in prefill - main_container = isvc_data["spec"]["prefill"]["template"]["containers"][0] - - # Set GPU resources to match tensor parallel size - apply_gpu_resources(main_container, tp_size) - - logging.info(f"Applied prefill GPU resources: {tp_size}") - - -def apply_kueue_configuration(isvc_data): - """ - Apply Kueue annotations and labels to the ISVC data - - Args: - isvc_data: The loaded YAML data structure - """ - kueue_config = config.project.get_config("tests.llmd.inference_service.kueue", {}) - - if not kueue_config.get("enabled", False): - logging.info("Kueue integration disabled - skipping kueue configuration") - return - - logging.info("Applying Kueue configuration to ISVC") - - # Get prefix for kueue labels/annotations - kueue_prefix = kueue_config.get("prefix") - - # Ensure metadata sections exist - if "metadata" not in isvc_data: - isvc_data["metadata"] = {} - if "labels" not in isvc_data["metadata"]: - isvc_data["metadata"]["labels"] = {} - if "annotations" not in isvc_data["metadata"]: - isvc_data["metadata"]["annotations"] = {} - - # Apply Kueue labels - kueue_labels = kueue_config.get("labels", {}) - for label_key, label_value in kueue_labels.items(): - full_label_key = f"{kueue_prefix}{label_key}" - isvc_data["metadata"]["labels"][full_label_key] = label_value - logging.info(f"Added Kueue label: {full_label_key}={label_value}") - - # Apply Kueue annotations - kueue_annotations = kueue_config.get("annotations", {}) - for annotation_key, annotation_value in kueue_annotations.items(): - full_annotation_key = f"{kueue_prefix}{annotation_key}" - isvc_data["metadata"]["annotations"][full_annotation_key] = annotation_value - logging.info(f"Added Kueue annotation: {full_annotation_key}={annotation_value}") - - # Apply Kueue annotations to router scheduler pod template - if ( - "spec" in isvc_data - and "router" in isvc_data["spec"] - and "scheduler" in isvc_data["spec"]["router"] - ): - scheduler_template = isvc_data["spec"]["router"]["scheduler"].get("template", {}) - - # Ensure metadata exists in scheduler template - if "metadata" not in scheduler_template: - scheduler_template["metadata"] = {} - if "annotations" not in scheduler_template["metadata"]: - scheduler_template["metadata"]["annotations"] = {} - - # Apply the same Kueue annotations to the scheduler pod template - for annotation_key, annotation_value in kueue_annotations.items(): - full_annotation_key = f"{kueue_prefix}{annotation_key}" - scheduler_template["metadata"]["annotations"][full_annotation_key] = annotation_value - logging.info( - f"Added Kueue annotation to scheduler template: {full_annotation_key}={annotation_value}" - ) - - # Update the scheduler template back to the data structure - isvc_data["spec"]["router"]["scheduler"]["template"] = scheduler_template - - # Calculate pod group total count: 1 scheduler + number of replicas - replicas = isvc_data.get("spec", {}).get("replicas", 1) - pod_group_total_count = 1 + replicas # 1 scheduler + replicas - isvc_data["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( - pod_group_total_count - ) - logging.info( - f"Set pod-group-total-count: {pod_group_total_count} (1 scheduler + {replicas} replicas)" - ) - - -def apply_model_configuration(isvc_data): - """ - Apply model URI and name configuration from config file - - Args: - isvc_data: The loaded YAML data structure - """ - model_key = config.project.get_config("tests.llmd.inference_service.model", None) - - if not model_key: - logging.info("No model configuration found - using defaults from YAML") - return - - logging.info(f"Applying model configuration for model reference: {model_key}") - - # Get model details from models section - models_config = config.project.get_config("models", {}) - if model_key not in models_config: - raise ValueError(f"Model '{model_key}' not found in models configuration") - - model_config = models_config[model_key] - - # Ensure spec.model section exists - if "spec" not in isvc_data: - isvc_data["spec"] = {} - if "model" not in isvc_data["spec"]: - isvc_data["spec"]["model"] = {} - - # Check if PVC prefetch is enabled - pvc_enabled = config.project.get_config( - "prepare.pvc.enabled", True - ) # Default to True for backward compatibility - - # Apply model URI based on configuration priority - if "uri" in model_config: - # Explicit URI in model config takes precedence - isvc_data["spec"]["model"]["uri"] = model_config["uri"] - logging.info(f"Set model URI from model config: {model_config['uri']}") - elif not pvc_enabled: - # PVC disabled - use model source directly as URI - if "source" in model_config: - isvc_data["spec"]["model"]["uri"] = model_config["source"] - logging.info(f"Set model URI from source (PVC disabled): {model_config['source']}") - else: - logging.warning(f"Model '{model_key}' has no source configured and PVC is disabled") - else: - # PVC enabled - construct PVC URI: pvc:/// - pvc_name = config.project.get_config("prepare.pvc.name") - uri = f"pvc://{pvc_name}/{model_key}" - isvc_data["spec"]["model"]["uri"] = uri - logging.info(f"Set model URI from PVC: {uri}") - - # Apply model name if configured - if "name" in model_config: - isvc_data["spec"]["model"]["name"] = model_config["name"] - logging.info(f"Set model name: {model_config['name']}") - - -def apply_vllm_args_configuration(isvc_data): - """ - Apply vLLM arguments configuration to ISVC containers - - Applies VLLM args to both main container and prefill container (for P/D deployments). - - Args: - isvc_data: The loaded YAML data structure - """ - vllm_args = config.project.get_config("tests.llmd.inference_service.vllm_args", []) - - if not vllm_args: - logging.info("No vLLM args configured") - return - - logging.info(f"Applying vLLM args: {vllm_args}") - - # Apply to main container only - _apply_vllm_args_to_container_section(isvc_data, "spec.template.containers", vllm_args, "main") - - -def _apply_vllm_args_to_container_section(isvc_data, container_path, vllm_args, container_name): - """ - Apply vLLM arguments to a specific container section - - Args: - isvc_data: The loaded YAML data structure - container_path: Dotted path to containers section (e.g., 'spec.template.containers') - vllm_args: List of vLLM arguments to apply - container_name: Name of the container to modify - """ - # Navigate to the container section - path_parts = container_path.split(".") - current = isvc_data - - # Create the path if it doesn't exist - for part in path_parts[:-1]: - if part not in current: - current[part] = {} - current = current[part] - - # Ensure containers array exists - containers_key = path_parts[-1] - if containers_key not in current: - current[containers_key] = [{"name": container_name}] - - containers = current[containers_key] - - # Find the target container - target_container = None - for container in containers: - if container.get("name") == container_name: - target_container = container - break - - if not target_container: - target_container = {"name": container_name} - containers.append(target_container) - - # Ensure env section exists - if "env" not in target_container: - target_container["env"] = [] - - # Find or create VLLM_ADDITIONAL_ARGS environment variable - vllm_args_env = None - for env_var in target_container["env"]: - if env_var.get("name") == "VLLM_ADDITIONAL_ARGS": - vllm_args_env = env_var - break - - if not vllm_args_env: - vllm_args_env = {"name": "VLLM_ADDITIONAL_ARGS", "value": ""} - target_container["env"].append(vllm_args_env) - - # Build vLLM arguments - current_args = vllm_args_env.get("value", "").strip() - - if isinstance(vllm_args, list): - new_args = vllm_args - else: - # Handle string format for backward compatibility - new_args = [vllm_args] - - # Combine existing and new arguments - if new_args: - if current_args: - combined_args = f"{current_args} {' '.join(new_args)}" - else: - combined_args = " ".join(new_args) - - vllm_args_env["value"] = combined_args - logging.info( - f"Set VLLM_ADDITIONAL_ARGS in {container_path}[{container_name}]: {combined_args}" - ) - - -def apply_max_model_len_configuration(isvc_data): - """ - Apply max-model-len configuration to ISVC containers - - Args: - isvc_data: The loaded YAML data structure - """ - max_model_len = config.project.get_config("tests.llmd.inference_service.max_model_len", None) - - if not max_model_len: - logging.debug("No max-model-len configured") - return - - logging.info(f"Applying max-model-len: {max_model_len}") - - # Apply to main container - _apply_vllm_args_to_container_section( - isvc_data, "spec.template.containers", ["--max-model-len", f"{max_model_len}"], "main" - ) - - # Apply to prefill container if this is a P/D deployment - if "spec" in isvc_data and "prefill" in isvc_data["spec"]: - logging.info("P/D deployment detected - applying max-model-len to prefill container") - _apply_vllm_args_to_container_section( - isvc_data, - "spec.prefill.template.containers", - [f"--max-model-len={max_model_len}"], - "main", - ) - - -def apply_image_pull_secrets_configuration(isvc_data): - """ - Apply image pull secrets configuration to ISVC templates - - Args: - isvc_data: The loaded YAML data structure - """ - image_pull_secrets = config.project.get_config( - "tests.llmd.inference_service.image_pull_secrets", None - ) - - if not image_pull_secrets: - logging.debug("No image pull secrets configured") - return - - logging.info(f"Applying image pull secrets: {image_pull_secrets}") - - # Apply to main template - isvc_data["spec"]["template"]["imagePullSecrets"] = [{"name": image_pull_secrets}] - logging.info(f"Set imagePullSecrets to '{image_pull_secrets}' in main template") - - # Apply to router scheduler template if this is a LLM-D deployment - if "router" in isvc_data["spec"] and "scheduler" in isvc_data["spec"]["router"]: - isvc_data["spec"]["router"]["scheduler"]["template"]["imagePullSecrets"] = [ - {"name": image_pull_secrets} - ] - logging.info(f"Set imagePullSecrets to '{image_pull_secrets}' in router.scheduler template") - - # Apply to prefill template if this is a P/D deployment - if "prefill" in isvc_data["spec"]: - isvc_data["spec"]["prefill"]["template"]["imagePullSecrets"] = [ - {"name": image_pull_secrets} - ] - logging.info(f"Set imagePullSecrets to '{image_pull_secrets}' in prefill template") - - -def apply_gpu_resources(main_container, gpu_count): - """ - Set GPU resource requests and limits based on tensor parallel size - - Args: - main_container: The main container definition - gpu_count: Number of GPUs needed (tensor_parallel_size) - """ - # Ensure resources section exists - if "resources" not in main_container: - main_container["resources"] = {} - if "requests" not in main_container["resources"]: - main_container["resources"]["requests"] = {} - if "limits" not in main_container["resources"]: - main_container["resources"]["limits"] = {} - - # Set GPU resources - main_container["resources"]["requests"]["nvidia.com/gpu"] = str(gpu_count) - main_container["resources"]["limits"]["nvidia.com/gpu"] = str(gpu_count) - - logging.info(f"Set GPU resources: nvidia.com/gpu={gpu_count} (for TP size {gpu_count})") - - -def apply_resource_configuration(isvc_data, model_key): - """ - Apply CPU and memory resource configuration to the ISVC template - - Checks for explicit test-level resource configuration first, then falls back - to model-specific resource requirements. - - Args: - isvc_data: The loaded YAML data structure - model_key: The model key to lookup model-specific resources - """ - # First, check for explicit test-level resource configuration - - models = config.project.get_config("models", {}) - model_data = models[model_key] - - model_resources = model_data.get("resources") - if not model_resources: - return - cpu_request = model_resources.get("cpu") - memory_request = model_resources.get("memory") - - if not cpu_request and not memory_request: - return - - # Navigate to spec.template.containers[0].resources.requests - try: - template = isvc_data["spec"]["template"] - containers = template["containers"] - main_container = containers[0] # Assume first container is main - - # Ensure resources.requests exists - if "resources" not in main_container: - main_container["resources"] = {} - if "requests" not in main_container["resources"]: - main_container["resources"]["requests"] = {} - - requests = main_container["resources"]["requests"] - - # Apply CPU request if configured - if cpu_request: - requests["cpu"] = str(cpu_request) - logging.info(f"Set CPU request: {cpu_request}") - - # Apply memory request if configured - if memory_request: - requests["memory"] = str(memory_request) - logging.info(f"Set memory request: {memory_request}") - - except (KeyError, IndexError) as e: - logging.error(f"Failed to apply resource configuration: {e}") - logging.error("Expected structure: spec.template.containers[0].resources.requests") - - -def apply_extra_properties(isvc_data): - """ - Apply extra properties from configuration to the ISVC - - Allows injecting arbitrary YAML properties using dotted-key notation. - Example config: - tests.llmd.inference_service.extra_properties: - spec.template.affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: NotIn - values: - - gf48e48 - - Args: - isvc_data: The loaded YAML data structure - """ - extra_properties = config.project.get_config( - "tests.llmd.inference_service.extra_properties", {} - ) - - if not extra_properties: - logging.debug("No extra properties configured") - return - - logging.info(f"Applying {len(extra_properties)} extra properties to ISVC") - - for dotted_key, value in extra_properties.items(): - _set_nested_property(isvc_data, dotted_key, value) - logging.info(f"Applied extra property: {dotted_key}") - - -def _set_nested_property(data, dotted_key, value): - """ - Set a nested property in a dict using dotted notation - - Args: - data: Dictionary to modify - dotted_key: Key path like 'spec.template.affinity' - value: Value to set - """ - keys = dotted_key.split(".") - current = data - - # Navigate to the parent of the final key - for key in keys[:-1]: - if key not in current: - current[key] = {} - current = current[key] - - # Set the final value - final_key = keys[-1] - current[final_key] = value - - -def apply_epp_configuration(isvc_data): - """ - Apply EPP (Endpoint Picker) configuration to the LLMISVC router container - - Adds the EPP configuration file path as a command line argument to the router container. - - Args: - isvc_data: The loaded YAML data structure - """ - epp_deploy = config.project.get_config("tests.llmd.inference_service.epp.deploy") - if epp_deploy is False: - logging.info("No EPP to deploy, delete spec.router") - del isvc_data["spec"]["router"] - return - elif epp_deploy == "default": - logging.info("EPP 'default' configuration. Keep the router untouched.") - return - elif epp_deploy != "custom": - raise ValueError(f"EPP configuration '{epp_deploy}' is not suppored.") - - epp_config_name = config.project.get_config("tests.llmd.inference_service.epp.config_file") - - if not epp_config_name: - raise ValueError(f"EPP configuration '{epp_deploy}' needs an EPP config file.") - - logging.info(f"Applying EPP configuration: {epp_config_name}") - - # Verify EPP configuration file exists - epp_config_path = TESTING_THIS_DIR / "epp-config" / epp_config_name - if not epp_config_path.exists(): - raise ValueError(f"EPP configuration file not found: {epp_config_path}") - - epp_value = epp_config_path.read_text() - - router_template = isvc_data["spec"]["router"]["scheduler"]["template"] - router_container = router_template["containers"][0] - - # Add the new EPP configuration argument - router_container["args"] = ["--config-text", epp_value] - - -def reshape_isvc(flavor, llmisvc_path, model_key): - """ - Reshape the ISVC YAML file based on configuration - - This method applies various modifications to the ISVC in a modular way: - 1. Flavor-specific modifications (routing, replicas, etc.) - 2. Kueue annotations and labels - 3. Model configuration - 4. vLLM arguments - 5. Extra properties (arbitrary YAML injection) - - Args: - flavor: The flavor string to apply - llmisvc_path: Path to the original ISVC YAML file - - Returns: - Path to the modified ISVC YAML file - """ - logging.info(f"Reshaping ISVC for flavor: {flavor}") - - # Load the YAML file - with open(llmisvc_path, "r") as f: - isvc_data = yaml.safe_load(f) - - # Apply modifications in order - apply_flavor_modifications(isvc_data, flavor) - apply_kueue_configuration(isvc_data) - apply_model_configuration(isvc_data) - apply_vllm_args_configuration(isvc_data) - apply_max_model_len_configuration(isvc_data) - apply_image_pull_secrets_configuration(isvc_data) - apply_resource_configuration(isvc_data, model_key) - apply_extra_properties(isvc_data) - apply_epp_configuration(isvc_data) - - # Save the modified file to ARTIFACT_DIR - output_path = env.ARTIFACT_DIR / llmisvc_path.name - try: - with open(output_path, "w") as f: - yaml.dump(isvc_data, f, default_flow_style=False) - except Exception as e: - raise ValueError(f"Failed to write reshaped ISVC file to {output_path}: {e}") - - logging.info(f"Reshaped ISVC saved to: {output_path}") - return output_path - - -def deploy_llm_inference_service(flavor, llmisvc_name, namespace, model_key): - """ - Deploys the LLM inference service - """ - - llmisvc_file = config.project.get_config("tests.llmd.inference_service.yaml_file") - - # Convert relative path to absolute - llmisvc_path = pathlib.Path(llmisvc_file) - if not llmisvc_path.is_absolute(): - llmisvc_path = TESTING_THIS_DIR / "llmisvcs" / llmisvc_path - - llmisvc_path = reshape_isvc(flavor, llmisvc_path, model_key) - - if config.project.get_config("tests.llmd.inference_service.skip_deployment"): - logging.info( - f"Skipping the deployment LLM inference service {llmisvc_name} in namespace {namespace}" - ) - else: - logging.info(f"Deploying LLM inference service {llmisvc_name} in namespace {namespace}") - - # Deploy the inference service - run.run_toolbox( - "llmd", - "deploy_llm_inference_service", - name=llmisvc_name, - namespace=namespace, - yaml_file=llmisvc_path, - ) - - return llmisvc_name, namespace, llmisvc_path - - -def get_llm_inference_url(llmisvc_name, namespace, flavor): - """ - Gets the URL of the deployed LLM inference service - """ - - logging.info(f"Getting LLM inference service URL for flavor: {flavor}") - - # Check if the LLM inference service has intelligent routing configured - scheduler_result = run.run( - f"oc get llmisvc/{llmisvc_name} -o jsonpath='{{.spec.router.scheduler}}' -n {namespace}", - capture_stdout=True, - check=False, - ) - - has_router_scheduler = ( - scheduler_result.returncode == 0 and scheduler_result.stdout.strip() != "" - ) - - # For services with intelligent routing, get the URL from status - if has_router_scheduler: - logging.info( - "LLM inference service has router scheduler - looking up the gateway URL from status" - ) - - # Get the LLMInferenceService status addresses - addresses_result = run.run( - f"oc get llminferenceservice {llmisvc_name} -n {namespace} " - f"-o jsonpath='{{.status.addresses}}'", - capture_stdout=True, - check=False, - ) - - if addresses_result.returncode != 0: - raise RuntimeError( - f"Failed to get LLMInferenceService addresses: {addresses_result.stderr}" - ) - - # Parse the addresses JSON to find the URL - gateway_name = config.project.get_config("tests.llmd.inference_service.gateway.name") - - try: - addresses = json.loads(addresses_result.stdout) - gateway_url = None - - for address in addresses: - if address.get("name") == gateway_name: - gateway_url = address.get("url") - break - - if not gateway_url: - raise RuntimeError( - f"{gateway_name} URL not found in LLMInferenceService status addresses" - ) - - logging.info(f"Intelligent-routing flavor - using {gateway_name} URL: {gateway_url}") - return gateway_url - - except (json.JSONDecodeError, KeyError) as e: - raise RuntimeError(f"Failed to parse LLMInferenceService addresses: {e}") - - # For simple flavors, we need to append the HTTPS port from the service - elif flavor.startswith("simple"): - logging.info("Simple flavor detected - looking up service port") - - service_name = f"{llmisvc_name}-kserve-workload-svc" - - # Get the HTTPS port from the service - port_result = run.run( - f"oc get svc {service_name} -n {namespace} " - """-o jsonpath='{.spec.ports[?(@.name==\"https\")].port}'""", - capture_stdout=True, - check=False, - ) - - if port_result.returncode != 0 or not port_result.stdout.strip(): - raise RuntimeError("Couldn't extract the SVC port :/") - https_port = port_result.stdout.strip() - endpoint_url = f"https://{service_name}.{namespace}.svc.cluster.local:{https_port}" - logging.info(f"Simple flavor - using port {https_port} from service") - - return endpoint_url - - else: - # For other flavors, use standard URL without port - service_name = f"{llmisvc_name}-kserve-workload-svc" - endpoint_url = f"https://{service_name}.{namespace}.svc.cluster.local" - logging.info(f"Other flavor - using standard URL without port: {endpoint_url}") - - return endpoint_url - - -def run_guidellm_benchmark(endpoint_url, llmisvc_name, namespace): - """ - Runs the Guidellm benchmark - """ - - if not config.project.get_config("tests.llmd.benchmarks.guidellm.enabled"): - return False - - logging.info("Running Guidellm benchmark") - - benchmark_name = config.project.get_config("tests.llmd.benchmarks.guidellm.name") - rate = config.project.get_config("tests.llmd.benchmarks.guidellm.rate") - - failed = False - - # Handle rate as list/tuple - iterate over each rate value - if isinstance(rate, (list, tuple)): - rate_values = rate - else: - rate_values = [rate] - - def apply_rate_scaleup(value, rate): - """ - Apply rate-based scaling to configuration values. - - Evaluates expressions like: - - "{10*rate}" with rate=32 -> "320" - - "prefix_count={2*rate}" with rate=32 -> "prefix_count=64" - """ - if not isinstance(value, str): - return value - - import re - - # Find all expressions in curly braces - pattern = r"\{([^}]+)\}" - - def evaluate_expression(match): - expression = match.group(1) - try: - # Create a safe evaluation context with only 'rate' variable - context = {"rate": rate} - result = eval(expression, {"__builtins__": {}}, context) - return str(result) - except Exception as e: - logging.warning( - f"Failed to evaluate expression '{expression}' with rate={rate}: {e}" - ) - return match.group(0) # Return original if evaluation fails - - # Replace all expressions with their evaluated results - return re.sub(pattern, evaluate_expression, value) - - for rate_value in rate_values: - try: - logging.info(f"Running Guidellm benchmark with rate: {rate_value}") - - # Create unique name for each rate if multiple rates - current_name = benchmark_name - if len(rate_values) > 1: - current_name = f"{benchmark_name}-rate-{rate_value}" - - # Construct guidellm arguments list - guidellm_args = [] - - # Add rate parameter - guidellm_args.append(f"--rate={rate_value}") - - # Iterate over tests.llmd.benchmarks.guidellm.args - # to generate --{arg_name.replace('_', '-')={apply_rate_scaleup(arg_value, rate_value)}} - guidellm_config_args = config.project.get_config("tests.llmd.benchmarks.guidellm.args") - for arg_name, arg_value in guidellm_config_args.items(): - if arg_value is None: # Guard: skip null values - continue - processed_value = apply_rate_scaleup(arg_value, rate_value) - guidellm_args.append(f"--{arg_name.replace('_', '-')}={processed_value}") - - # Add tests.llmd.benchmarks.guidellm.extra_args - extra_args = config.project.get_config("tests.llmd.benchmarks.guidellm.extra_args") - for extra_arg_name, extra_arg_value in extra_args.items(): - if extra_arg_value is None: # Guard: skip null values - continue - processed_value = apply_rate_scaleup(extra_arg_value, rate_value) - guidellm_args.append(f"--{extra_arg_name.replace('_', '-')}={processed_value}") - - # Construct image reference - image_name = config.project.get_config("tests.llmd.benchmarks.guidellm.image.name") - image_version = config.project.get_config( - "tests.llmd.benchmarks.guidellm.image.version" - ) - - suffix = f"_rate{rate_value}" if len(rate_values) > 1 else None - - run.run_toolbox( - "llmd", - "run_guidellm_benchmark", - endpoint_url=endpoint_url, - name=current_name, - namespace=namespace, - image=image_name, - version=image_version, - guidellm_args=guidellm_args, - run_as_root=config.project.get_config("security.run_as_root"), - artifact_dir_suffix=suffix, - ) - - logging.info(f"Guidellm benchmark completed successfully for rate: {rate_value}") - - except Exception as e: - logging.exception(f"Guidellm benchmark failed for rate {rate_value}: {e}") - failed = True - - return failed - - -def capture_llm_inference_service_state(llmisvc_name, namespace): - """ - Captures the state of the LLM inference service - """ - - logging.info("Capturing LLM inference service state") - - try: - run.run_toolbox( - "llmd", - "capture_isvc_state", - llmisvc_name=llmisvc_name, - namespace=namespace, - mute_stdout=True, - ) - - logging.info("LLM inference service state captured successfully") - - except Exception as e: - logging.error(f"Failed to capture LLM inference service state: {e}") - - -def cleanup_llm_inference_resources(): - """ - Clean up all llminferenceservice resources in the namespace before testing. - Fails the test if cleanup is not successful. - """ - - namespace = config.project.get_config("tests.llmd.namespace") - logging.info(f"Cleaning up all jobs resources in namespace {namespace}") - result = run.run(f"oc delete jobs --all -n {namespace}", capture_stdout=True) - - logging.info(f"Cleaning up all llminferenceservice resources in namespace {namespace}") - - # Delete all llminferenceservice resources in the namespace - logging.info("Deleting all llminferenceservice resources") - run.run(f"oc delete llminferenceservice --all -n {namespace} --wait=true --timeout=180s") - - # Verify no llminferenceservice resources remain - logging.info("Verifying no llminferenceservice resources remain") - for i in range(6): # Check up to 6 times with 10 second intervals - result = run.run( - f"oc get llminferenceservice -n {namespace} --no-headers", capture_stdout=True - ) - - if not result.stdout.strip(): - logging.info("No llminferenceservice resources found - cleanup successful") - break - else: - remaining_services = result.stdout.strip().split("\n") - remaining_count = len([s for s in remaining_services if s.strip()]) - logging.info(f"Still found {remaining_count} llminferenceservice resources, waiting...") - - if i == 5: # Last iteration - logging.error( - f"Failed to clean up llminferenceservice resources after 60 seconds. {remaining_count} resources still exist:" - ) - for service in remaining_services: - if service.strip(): - logging.error(f" - {service}") - raise RuntimeError( - f"Cannot proceed with test - {remaining_count} llminferenceservice resources still exist in namespace {namespace}" - ) - - time.sleep(10) - - logging.info("LLM inference service cleanup completed successfully") - - -def ensure_gpu_nodes_available(): - """ - Ensures that there are GPU nodes available in the cluster. - This function assumes prepare_gpu() has already been called. - """ - - logging.info("Verifying GPU nodes are available in the cluster") - - try: - result = run.run( - "oc get nodes -l nvidia.com/gpu.present=true --no-headers", capture_stdout=True - ) - - if result.returncode != 0: - raise RuntimeError(f"Failed to check GPU nodes: {result.stderr}") - - gpu_nodes = result.stdout.strip().split("\n") if result.stdout.strip() else [] - has_gpu_nodes = gpu_nodes and not (len(gpu_nodes) == 1 and gpu_nodes[0] == "") - - if not has_gpu_nodes: - raise RuntimeError( - "No GPU nodes found in the cluster. GPU nodes are required for LLM inference testing. Ensure prepare_gpu() was called." - ) - else: - logging.info(f"Found {len(gpu_nodes)} GPU node(s) in the cluster") - for node in gpu_nodes: - node_name = node.split()[0] - logging.info(f" - {node_name}") - - except Exception as e: - logging.error(f"GPU node validation failed: {e}") - raise - - -def test_llm_inference_simple(endpoint_url, llmisvc_name, namespace, model_name): - """ - Runs a simple test against the LLM inference service using oc rsh - """ - - logging.info("Running simple LLM inference test from inside cluster") - - # Construct the service URL - deployment_name = f"{llmisvc_name}-kserve" - - logging.info(f"Testing URL: {endpoint_url}") - - # Test with a simple completion request - test_payload = { - "model": model_name, - "prompt": "San Francisco is a", - "max_tokens": 50, - "temperature": 0.7, - } - - try: - # Convert payload to JSON string for inline use - payload_json = json.dumps(test_payload) - - remaining_tries = 30 - DELAY = 10 - result = None - - while remaining_tries != 0: - # Execute curl inside the pod with inline JSON - result = run.run( - f""" - oc rsh -n {namespace} -c main deploy/{deployment_name} \\ - curl -k -sSf "{endpoint_url}/v1/completions" \\ - -H "Content-Type: application/json" \\ - -d '{payload_json}' - """, - capture_stdout=True, - check=False, - ) - - logging.info(f"Request result: {result.returncode}") - if result.returncode == 0: - break - logging.info(f"Waiting {DELAY}s before retrying ...") - time.sleep(DELAY) - remaining_tries -= 1 - - if result.returncode != 0: - logging.error(f"Request failed :/") - return False - - response = json.loads(result.stdout) - if "choices" not in response or not len(response["choices"]): - logging.error(f"Invalid response format: {result.stdout}") - return False - - completion = response["choices"][0]["text"] - logging.info( - f"Simple LLM test successful. Completion: {test_payload['prompt']} ... {completion[:100]}" - ) - - return True - - except Exception as e: - logging.error(f"Simple LLM test failed: {e}") - return False - - -def conditional_scale_up(): - """ - Conditionally scales up GPU nodes if scale_up preset is enabled and no GPU nodes exist - """ - auto_scale = config.project.get_config("prepare.cluster.nodes.auto_scale") - if not auto_scale: - return - - logging.info("Auto scale enabled, checking for existing GPU nodes") - - try: - result = run.run( - "oc get nodes -l nvidia.com/gpu.present=true --no-headers", capture_stdout=True - ) - gpu_nodes = result.stdout.strip().split("\n") if result.stdout.strip() else [] - has_gpu_nodes = gpu_nodes and not (len(gpu_nodes) == 1 and gpu_nodes[0] == "") - - if not has_gpu_nodes: - logging.info("No GPU nodes found, scaling up cluster") - - prepare_llmd.scale_up() - else: - logging.info(f"Found {len(gpu_nodes)} existing GPU node(s), skipping scale up") - - except Exception as e: - logging.error(f"Failed to check/scale GPU nodes: {e}") - raise - - -def conditional_scale_down(): - """ - Conditionally scales down GPU nodes if scale_up preset is enabled - """ - auto_scale_down = config.project.get_config("prepare.cluster.nodes.auto_scale_down_on_exit") - if not auto_scale_down: - return - - node_instance_type = config.project.get_config("prepare.cluster.nodes.instance_type") - if not node_instance_type: - logging.warning("Node instance type not configured, skipping scale down") - return - - logging.info("Auto scale down enabled, scaling down GPU nodes to 0") - - try: - run.run_toolbox( - "cluster", "set_scale", instance_type=node_instance_type, scale=0, mute_stdout=True - ) - logging.info("GPU nodes scaled down successfully") - except Exception as e: - logging.error(f"Failed to scale down GPU nodes: {e}") - - -def matbench_run_one(): - """ - Runs one test as part of a MatrixBenchmark benchmark - """ - - logging.info("Running MatrixBenchmark test") - - # Deploy and test (test() now handles its own GPU scaling) - failed = test() - - if not failed: - logging.info("MatrixBenchmark test completed successfully") - else: - logging.error("MatrixBenchmark test failed") - - return failed diff --git a/projects/llm_d_legacy/toolbox/llmd.py b/projects/llm_d_legacy/toolbox/llmd.py deleted file mode 100644 index c236e2314..000000000 --- a/projects/llm_d_legacy/toolbox/llmd.py +++ /dev/null @@ -1,136 +0,0 @@ -import sys -import logging - -from projects.legacy.library.ansible_toolbox import ( - RunAnsibleRole, - AnsibleRole, - AnsibleMappedParams, - AnsibleConstant, - AnsibleSkipConfigGeneration, -) - - -class Llmd: - """ - Commands and utilities for the LLM-D toolbox - """ - - @AnsibleRole("llmd_deploy_gateway") - @AnsibleMappedParams - def deploy_gateway( - self, - name="openshift-ai-inference", - gateway_class="data-science-gateway-class", - namespace="openshift-ingress", - ): - """ - Deploys a GatewayClass and Gateway object - - Default gateway class is created by the DSCI -> GatewayConfig/default-gateway -> GatewayClass/data-science-gateway-class - - Args: - name: Name of the gateway to deploy - gateway_class: Name of the gateway class to deploy - namespace: Namespace where the gateway will be deployed - """ - - if name != "openshift-ai-inference": - logging.error("Currently the gateway name must be 'openshift-ai-inference'") - raise SystemExit(1) - - return RunAnsibleRole(locals()) - - @AnsibleRole("llmd_deploy_llm_inference_service") - @AnsibleMappedParams - def deploy_llm_inference_service(self, name, namespace, yaml_file): - """ - Deploys an LLM InferenceService from a YAML file - - Args: - name: Name of the inference service to deploy - namespace: Namespace to deploy the inference service in - yaml_file: Path to the YAML file containing the LLMInferenceService - """ - - return RunAnsibleRole(locals()) - - @AnsibleRole("llmd_run_guidellm_benchmark") - @AnsibleMappedParams - def run_guidellm_benchmark( - self, - endpoint_url, - name="guidellm-benchmark", - namespace="", - image="ghcr.io/vllm-project/guidellm", - version="v0.6.0", - timeout=900, - pvc_size="1Gi", - guidellm_args=[], - run_as_root=False, - ): - """ - Runs a Guidellm benchmark job against the LLM inference service - - Args: - endpoint_url: Endpoint URL for the LLM inference service to benchmark - name: Name of the benchmark job - namespace: Namespace to run the benchmark job in (empty string auto-detects current namespace) - image: Container image for the benchmark - version: Version tag for the benchmark image - timeout: Timeout in seconds to wait for job completion - pvc_size: Size of the PersistentVolumeClaim for storing results - guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) - run_as_root: Run the GuideLLM container as root user - """ - - return RunAnsibleRole(locals()) - - @AnsibleRole("llmd_capture_isvc_state") - @AnsibleMappedParams - def capture_isvc_state(self, llmisvc_name, namespace=""): - """ - Captures all relevant objects and state for an LLMInferenceService - - Args: - llmisvc_name: Name of the LLMInferenceService to capture - namespace: Namespace of the LLMInferenceService (empty string auto-detects current namespace) - """ - - return RunAnsibleRole(locals()) - - @AnsibleRole("storage_download_to_pvc") - @AnsibleMappedParams - def download_to_pvc( - self, - name, - source, - pvc_name, - namespace, - creds="", - storage_dir="/", - clean_first=False, - pvc_access_mode="ReadWriteOnce", - pvc_size="80Gi", - pvc_storage_class_name=None, - image="registry.access.redhat.com/ubi9/ubi", - run_as_root=False, - ): - """ - Downloads the a dataset into a PVC of the cluster - - Args: - name: Name of the data source - source: URL of the source data - pvc_name: Name of the PVC that will be create to store the dataset files. - namespace: Name of the namespace in which the PVC will be created - creds: Path to credentials to use for accessing the dataset. - clean_first: if True, clears the storage directory before downloading. - storage_dir: the path where to store the downloaded files, in the PVC - pvc_access_mode: the access mode to request when creating the PVC - pvc_size: the size of the PVC to request, when creating the PVC - pvc_storage_class_name: the name of the storage class to pass when creating the PVC - image: the image to use for running the download Pod - run_as_root: Run the download container as root user - """ - - return RunAnsibleRole(locals()) diff --git a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/defaults/main/config.yml b/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/defaults/main/config.yml deleted file mode 100644 index 215249af3..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/defaults/main/config.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Llmd.capture_isvc_state - -# Parameters -# Name of the LLMInferenceService to capture -# Mandatory value -llmd_capture_isvc_state_llmisvc_name: - -# Namespace of the LLMInferenceService (empty string auto-detects current namespace) -llmd_capture_isvc_state_namespace: - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/tasks/main.yml b/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/tasks/main.yml deleted file mode 100644 index db4c796b8..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/tasks/main.yml +++ /dev/null @@ -1,132 +0,0 @@ ---- -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/" - state: directory - mode: '0755' - -- name: Create the logs directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/logs/" - state: directory - mode: '0755' - -- name: Get current timestamp - command: date -Iseconds - register: capture_timestamp - -- name: Get current namespace if not specified - command: oc project -q - register: current_namespace_result - when: llmd_capture_isvc_state_namespace is none or llmd_capture_isvc_state_namespace == "" - -- name: Set the target namespace - set_fact: - target_namespace: "{{ current_namespace_result.stdout if (llmd_capture_isvc_state_namespace is none or llmd_capture_isvc_state_namespace == '') else llmd_capture_isvc_state_namespace }}" - -- name: Capture the LLMInferenceService definition - shell: - oc get llminferenceservice "{{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.yaml" - ignore_errors: true - -- name: Capture LLMInferenceService status in JSON for easier parsing - shell: - oc get llminferenceservice "{{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -ojson > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.json" - ignore_errors: true - -- name: Capture all pods related to the LLMInferenceService - shell: - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.pods.yaml" - ignore_errors: true - -- name: Capture deployments related to the LLMInferenceService - shell: - oc get deployments \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.deployments.yaml" - ignore_errors: true - -- name: Capture replicasets related to the LLMInferenceService - shell: - oc get replicasets \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.replicasets.yaml" - ignore_errors: true - -- name: Capture all pods in the namespace with wide output - shell: - oc get pods -owide \ - -n "{{ target_namespace }}" \ - > "{{ artifact_extra_logs_dir }}/artifacts/namespace.pods.status" - ignore_errors: true - -- name: Capture all services in the namespace - shell: - oc get svc \ - -n "{{ target_namespace }}" \ - > "{{ artifact_extra_logs_dir }}/artifacts/namespace.services.status" - ignore_errors: true - -- name: Capture ServiceMonitors for monitoring - shell: - oc get servicemonitor \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.servicemonitors.yaml" - ignore_errors: true - -- name: Capture PodMonitors for monitoring - shell: - oc get podmonitor \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.podmonitors.yaml" - ignore_errors: true - -- name: Capture logs from LLMInferenceService pods - shell: | - for pod in $(oc get pods \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -o jsonpath='{.items[*].metadata.name}'); do - oc logs "$pod" -n "{{ target_namespace }}" --all-containers=true > "{{ artifact_extra_logs_dir }}/artifacts/logs/$pod.log" 2>&1 || true - done - ignore_errors: true - -- name: Capture previous logs from LLMInferenceService pods if available - shell: | - for pod in $(oc get pods \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -o jsonpath='{.items[*].metadata.name}'); do - oc logs "$pod" -n "{{ target_namespace }}" --previous --all-containers=true > "{{ artifact_extra_logs_dir }}/artifacts/logs/$pod.previous.log" 2>&1 || true - done - ignore_errors: true - -- name: Capture describe output for the LLMInferenceService - shell: - oc describe llminferenceservice "{{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - > "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.describe.txt" - ignore_errors: true - -- name: Capture describe output for related pods - shell: | - for pod in $(oc get pods \ - -l "app.kubernetes.io/name={{ llmd_capture_isvc_state_llmisvc_name }}" \ - -n "{{ target_namespace }}" \ - -o jsonpath='{.items[*].metadata.name}'); do - echo "=== Describe for pod: $pod ===" >> "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.pods.describe.txt" - oc describe pod "$pod" -n "{{ target_namespace }}" >> "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.pods.describe.txt" 2>&1 || true - echo "" >> "{{ artifact_extra_logs_dir }}/artifacts/llminferenceservice.pods.describe.txt" - done - ignore_errors: true diff --git a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/vars/main/resources.yml b/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/vars/main/resources.yml deleted file mode 100644 index 52e4c6f7b..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_capture_isvc_state/vars/main/resources.yml +++ /dev/null @@ -1 +0,0 @@ -# No additional resources needed for state capture diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/defaults/main/config.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/defaults/main/config.yml deleted file mode 100644 index da5d853d5..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/defaults/main/config.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Llmd.deploy_gateway - -# Parameters -# Name of the gateway to deploy -llmd_deploy_gateway_name: openshift-ai-inference - -# Name of the gateway class to deploy -llmd_deploy_gateway_gateway_class: data-science-gateway-class - -# Namespace where the gateway will be deployed -llmd_deploy_gateway_namespace: openshift-ingress - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/files/.keep b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/meta/main.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/meta/main.yml deleted file mode 100644 index bd0638df3..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: check_deps diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/tasks/main.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/tasks/main.yml deleted file mode 100644 index 64983e9c8..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/tasks/main.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -- name: Create the src directory - file: - path: "{{ artifact_extra_logs_dir }}/src/" - state: directory - mode: '0755' - -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/" - state: directory - mode: '0755' - -- name: Check if the gateway already exists - command: - oc get gateway "{{ llmd_deploy_gateway_name }}" -n "{{ llmd_deploy_gateway_namespace }}" --ignore-not-found - register: has_gateway_cmd - -- name: Save the gateway - shell: - oc get gateway "{{ llmd_deploy_gateway_name }}" -n "{{ llmd_deploy_gateway_namespace }}" -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/gateway.old.yaml" - when: has_gateway_cmd.stdout | length > 0 - -- name: Delete the old gateway - command: - oc delete gateway "{{ llmd_deploy_gateway_name }}" -n "{{ llmd_deploy_gateway_namespace }}" - when: has_gateway_cmd.stdout | length > 0 - -- name: Wait for the gateway service to disappear - command: - oc get svc -l "gateway.networking.k8s.io/gateway-name={{ llmd_deploy_gateway_name }}" --all-namespaces --no-headers - register: gateway_svc_check - until: gateway_svc_check.stdout == "" - retries: 20 - delay: 5 - when: has_gateway_cmd.stdout | length > 0 - -- name: Prepare the Gateway resource - template: - src: "{{ gateway_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/gateway.yaml" - mode: '0700' - -- name: Create the gateway - command: - oc create -f "{{ artifact_extra_logs_dir }}/src/gateway.yaml" - -- name: Wait for gateway readiness and capture artifacts - block: - - name: Wait for the gateway service to appear - command: - oc get svc -l "gateway.networking.k8s.io/gateway-name={{ llmd_deploy_gateway_name }}" --all-namespaces --no-headers - register: gateway_svc_appear - until: gateway_svc_appear.stdout != "" - retries: 20 - delay: 5 - - - name: Wait for the gateway to be programmed - command: - oc get gateway "{{ llmd_deploy_gateway_name }}" -n "{{ llmd_deploy_gateway_namespace }}" -o jsonpath='{.status.conditions[?(@.type=="Programmed")].status}' - register: gateway_programmed_status - until: gateway_programmed_status.stdout == "True" - retries: 20 - delay: 10 - - always: - - name: Capture the final gateway YAML - shell: - oc get gateway "{{ llmd_deploy_gateway_name }}" -n "{{ llmd_deploy_gateway_namespace }}" -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/gateway.final.yaml" - - - name: Capture the gateway service YAML - shell: - oc get svc -l "gateway.networking.k8s.io/gateway-name={{ llmd_deploy_gateway_name }}" --all-namespaces -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/gateway.service.yaml" diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/templates/gateway.yaml.j2 b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/templates/gateway.yaml.j2 deleted file mode 100644 index d522651d5..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/templates/gateway.yaml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: "{{ llmd_deploy_gateway_name }}" - namespace: "{{ llmd_deploy_gateway_namespace }}" -spec: - gatewayClassName: "{{ llmd_deploy_gateway_gateway_class }}" - listeners: - - name: http - port: 80 - protocol: HTTP - allowedRoutes: - namespaces: - from: All diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/vars/main/resources.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/vars/main/resources.yml deleted file mode 100644 index dff599bf2..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_gateway/vars/main/resources.yml +++ /dev/null @@ -1 +0,0 @@ -gateway_template: templates/gateway.yaml.j2 diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/defaults/main/config.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/defaults/main/config.yml deleted file mode 100644 index a99e60608..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/defaults/main/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Llmd.deploy_llm_inference_service - -# Parameters -# Name of the inference service to deploy -# Mandatory value -llmd_deploy_llm_inference_service_name: - -# Namespace to deploy the inference service in -# Mandatory value -llmd_deploy_llm_inference_service_namespace: - -# Path to the YAML file containing the LLMInferenceService -# Mandatory value -llmd_deploy_llm_inference_service_yaml_file: - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/tasks/main.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/tasks/main.yml deleted file mode 100644 index 0cc8112be..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/tasks/main.yml +++ /dev/null @@ -1,165 +0,0 @@ ---- -- name: Create the src directory - file: - path: "{{ artifact_extra_logs_dir }}/src/" - state: directory - mode: '0755' - -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/" - state: directory - mode: '0755' - -- name: Check if the LLM inference service already exists - command: - oc get llminferenceservice "{{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - --ignore-not-found - register: has_llm_inference_service_cmd - -- name: Save the LLM inference service - shell: - oc get llminferenceservice "{{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.old.yaml" - when: has_llm_inference_service_cmd.stdout | length > 0 - -- name: Delete the old LLM inference service - command: - oc delete llminferenceservice "{{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - --ignore-not-found - -- name: Wait for all old LLM inference service pods to be deleted - shell: | - set -o pipefail; - oc get pods \ - -l "app.kubernetes.io/component=llminferenceservice-workload,app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - --no-headers | wc -l - register: old_pods_count - until: old_pods_count.stdout | int == 0 - retries: 30 - delay: 10 - failed_when: false - -- name: Copy the LLM InferenceService YAML file - copy: - src: "{{ llmd_deploy_llm_inference_service_yaml_file }}" - dest: "{{ artifact_extra_logs_dir }}/src/llm_inference_service.yaml" - mode: '0700' - -- name: Set the name and namespace in the LLM inference service YAML - shell: | - yq -Yi '.metadata.name = "{{ llmd_deploy_llm_inference_service_name }}"' \ - "{{ artifact_extra_logs_dir }}/src/llm_inference_service.yaml" - yq -Yi '.metadata.namespace = "{{ llmd_deploy_llm_inference_service_namespace }}"' \ - "{{ artifact_extra_logs_dir }}/src/llm_inference_service.yaml" - -- name: Create the LLM inference service - command: - oc create -f "{{ artifact_extra_logs_dir }}/src/llm_inference_service.yaml" - -- name: Wait for LLM inference service readiness and capture artifacts - block: - - name: Wait for the LLM inference service pods to appear - command: - oc get pods \ - -l "app.kubernetes.io/component=llminferenceservice-workload,app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - --no-headers - register: llm_inference_service_pods_appear - until: llm_inference_service_pods_appear.stdout != "" - retries: 20 - delay: 5 - - - name: Wait for the LLM inference service pods to be scheduled (not Pending) - shell: | - set -o pipefail; - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -o json | jq -r '.items[] | select(.status.phase == "Pending") | .metadata.name' | wc -l - register: llm_pods_pending - until: llm_pods_pending.stdout | int == 0 - retries: 360 - delay: 10 - failed_when: llm_pods_pending.stdout | int > 0 - - - name: Wait for the LLM inference service pods to be running (images pulled and containers started) - shell: | - set -o pipefail; - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -o json | jq -r '.items[] | select(.status.phase != "Running" and .status.phase != "Succeeded") | .metadata.name' | wc -l - register: llm_pods_not_running - until: llm_pods_not_running.stdout | int == 0 - retries: 60 - delay: 15 - failed_when: false - - - name: Debug - show pod status during startup wait - command: - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -o wide - when: llm_pods_not_running.stdout | int > 0 - - - name: Wait for the LLM inference service to be ready - command: - oc get llminferenceservice "{{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' - register: llm_inference_service_ready_status - until: llm_inference_service_ready_status.stdout == "True" - retries: 30 - delay: 10 - - always: - - name: Capture the final LLM inference service YAML - shell: - oc get llminferenceservice "{{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.final.yaml" - failed_when: false - - - name: Capture pod status with oc get pods - shell: - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -o wide \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.pods.status" - failed_when: false - - - name: Capture deployment YAML - shell: - oc get deployments \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.deployments.yaml" - failed_when: false - - - name: Capture replicaset YAML - shell: - oc get replicasets \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.replicasets.yaml" - failed_when: false - - - name: Capture the LLM inference service pods YAML - shell: - oc get pods \ - -l "app.kubernetes.io/name={{ llmd_deploy_llm_inference_service_name }}" \ - -n "{{ llmd_deploy_llm_inference_service_namespace }}" \ - -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/llm_inference_service.pods.yaml" - failed_when: false diff --git a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/vars/main/resources.yml b/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/vars/main/resources.yml deleted file mode 100644 index 33e77da04..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_deploy_llm_inference_service/vars/main/resources.yml +++ /dev/null @@ -1 +0,0 @@ -# No template needed - using direct YAML file diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/defaults/main/config.yml b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/defaults/main/config.yml deleted file mode 100644 index 74ba19109..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/defaults/main/config.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Llmd.run_guidellm_benchmark - -# Parameters -# Endpoint URL for the LLM inference service to benchmark -# Mandatory value -llmd_run_guidellm_benchmark_endpoint_url: - -# Name of the benchmark job -llmd_run_guidellm_benchmark_name: guidellm-benchmark - -# Namespace to run the benchmark job in (empty string auto-detects current namespace) -llmd_run_guidellm_benchmark_namespace: - -# Container image for the benchmark -llmd_run_guidellm_benchmark_image: ghcr.io/vllm-project/guidellm - -# Version tag for the benchmark image -llmd_run_guidellm_benchmark_version: v0.6.0 - -# Timeout in seconds to wait for job completion -llmd_run_guidellm_benchmark_timeout: 900 - -# Size of the PersistentVolumeClaim for storing results -llmd_run_guidellm_benchmark_pvc_size: 1Gi - -# List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) -llmd_run_guidellm_benchmark_guidellm_args: [] - -# Run the GuideLLM container as root user -llmd_run_guidellm_benchmark_run_as_root: false - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/tasks/main.yml b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/tasks/main.yml deleted file mode 100644 index dc3a4dd03..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/tasks/main.yml +++ /dev/null @@ -1,146 +0,0 @@ ---- -- name: Create the src directory - file: - path: "{{ artifact_extra_logs_dir }}/src/" - state: directory - mode: '0755' - -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/" - state: directory - mode: '0755' - -- name: Get current namespace if not specified - command: oc project -q - register: current_namespace_result - when: llmd_run_guidellm_benchmark_namespace == "" - -- name: Set the target namespace - set_fact: - target_namespace: "{{ current_namespace_result.stdout if llmd_run_guidellm_benchmark_namespace == '' else llmd_run_guidellm_benchmark_namespace }}" - -- name: Delete the old guidellm benchmark job and PVC - command: - oc delete job,pvc "{{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" --ignore-not-found - -- name: Create the guidellm benchmark PVC YAML - template: - src: "{{ guidellm_benchmark_pvc_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/guidellm_benchmark_pvc.yaml" - mode: '0700' - -- name: Create the guidellm benchmark PVC - command: - oc create -f "{{ artifact_extra_logs_dir }}/src/guidellm_benchmark_pvc.yaml" - -- name: Set the benchmark endpoint URL - set_fact: - benchmark_endpoint_url: "{{ llmd_run_guidellm_benchmark_endpoint_url }}" - -- name: Create the guidellm benchmark job YAML - template: - src: "{{ guidellm_benchmark_job_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/guidellm_benchmark_job.yaml" - mode: '0700' - -- name: Create the guidellm benchmark job - command: - oc create -f "{{ artifact_extra_logs_dir }}/src/guidellm_benchmark_job.yaml" - -- name: Run guidellm benchmark and capture artifacts - block: - - name: Compute job timeout end time - shell: echo $(($(date +%s) + {{ llmd_run_guidellm_benchmark_timeout }})) - register: job_end_time_result - - - name: Get current time and end time for debug - shell: | - end_epoch={{ job_end_time_result.stdout }} - current_human=$(date) - end_human=$(date -d @$end_epoch) - echo "Current: $current_human" - echo "End: $end_human" - echo "Timeout: {{ llmd_run_guidellm_benchmark_timeout }}s" - - - name: Wait for the guidellm benchmark job to complete or fail - shell: | - set -o pipefail; - oc get job "{{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" --no-headers | awk '{print $2; print "Job duration:", $4 > "/dev/stderr"}' - register: job_status_result - until: job_status_result.stdout in ['Complete', 'Failed'] - retries: 900 - delay: 30 - failed_when: false - - - name: Check final job status and fail if not completed successfully - fail: - msg: "Guidellm benchmark job failed or timed out (final status: {{ job_status_result.stdout }})" - when: job_status_result.stdout != 'Complete' - - always: - - name: Capture artifacts - block: - - name: Capture the final guidellm benchmark job YAML - shell: - oc get job "{{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/guidellm_benchmark_job.final.yaml" - - - name: Capture the guidellm benchmark job pods YAML - shell: - oc get pods -l "job-name={{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" -oyaml \ - > "{{ artifact_extra_logs_dir }}/artifacts/guidellm_benchmark_job.pods.yaml" - - - name: Capture the guidellm benchmark job logs - shell: - oc logs job/{{ llmd_run_guidellm_benchmark_name }} -n "{{ target_namespace }}" \ - > "{{ artifact_extra_logs_dir }}/artifacts/guidellm_benchmark_job.logs" - always: - - name: Create the results subdirectory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts/results/" - state: directory - mode: '0755' - - - name: Get the node where the GuideLLM job pod ran - shell: | - set -o pipefail; - oc get pods -l "job-name={{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" --no-headers -o custom-columns=":spec.nodeName" | head -1 - register: guidellm_node_result - failed_when: false - - - name: Set the target node for helper pod - set_fact: - helper_pod_node: "{{ guidellm_node_result.stdout | trim }}" - - - name: Create the copy helper pod YAML - template: - src: "{{ copy_helper_pod_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/copy_helper_pod.yaml" - mode: '0700' - - - name: Create the copy helper pod - command: - oc create -f "{{ artifact_extra_logs_dir }}/src/copy_helper_pod.yaml" - - - name: Retrieve the test results locally - block: - - name: Wait for copy helper pod to be ready - shell: | - oc wait --for=condition=Ready pod/{{ llmd_run_guidellm_benchmark_name }}-copy -n "{{ target_namespace }}" --timeout=60s - - - name: Copy benchmarks.json from PVC to artifacts directory - shell: | - oc exec {{ llmd_run_guidellm_benchmark_name }}-copy -n "{{ target_namespace }}" -- cat /results/benchmarks.json > "{{ artifact_extra_logs_dir }}/artifacts/results/benchmarks.json" 2>/dev/null || echo "Warning: benchmarks.json not found in PVC" - always: - - name: Delete the copy helper pod - command: - oc delete pod "{{ llmd_run_guidellm_benchmark_name }}-copy" -n "{{ target_namespace }}" --ignore-not-found - - - name: Delete the guidellm benchmark job - command: - oc delete job "{{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" --ignore-not-found - - - name: Delete the guidellm benchmark PVC - command: - oc delete pvc "{{ llmd_run_guidellm_benchmark_name }}" -n "{{ target_namespace }}" --ignore-not-found diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/copy_helper_pod.yaml.j2 b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/copy_helper_pod.yaml.j2 deleted file mode 100644 index 12bf53f72..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/copy_helper_pod.yaml.j2 +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: {{ llmd_run_guidellm_benchmark_name }}-copy - namespace: {{ target_namespace }} -spec: - restartPolicy: Never -{% if helper_pod_node and helper_pod_node != "" %} - nodeName: "{{ helper_pod_node }}" -{% endif %} - initContainers: - - name: permission-fixer - image: "{{ llmd_run_guidellm_benchmark_image }}:{{ llmd_run_guidellm_benchmark_version }}" - command: ["/bin/sh", "-c", "chmod 755 /results && chown -R 1001:1001 /results || true"] - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: true - volumeMounts: - - name: results - mountPath: /results - containers: - - name: copy-helper - image: "{{ llmd_run_guidellm_benchmark_image }}:{{ llmd_run_guidellm_benchmark_version }}" - command: ["/bin/sleep", "300"] - securityContext: - runAsUser: 1001 - runAsNonRoot: true - allowPrivilegeEscalation: false - volumeMounts: - - name: results - mountPath: /results - volumes: - - name: results - persistentVolumeClaim: - claimName: "{{ llmd_run_guidellm_benchmark_name }}" diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_job.yaml.j2 b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_job.yaml.j2 deleted file mode 100644 index f67bdf581..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_job.yaml.j2 +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: "{{ llmd_run_guidellm_benchmark_name }}" - namespace: "{{ target_namespace }}" -spec: - backoffLimit: 0 - template: - spec: - serviceAccountName: default - restartPolicy: Never - containers: - - name: guidellm - image: "{{ llmd_run_guidellm_benchmark_image }}:{{ llmd_run_guidellm_benchmark_version }}" - command: - - /opt/app-root/bin/guidellm - args: - - benchmark - - run - - --target={{ benchmark_endpoint_url }} -{% for arg in llmd_run_guidellm_benchmark_guidellm_args %} - - {{ arg }} -{% endfor %} - - --outputs=json - env: - - name: USER # version 0.6.0-pr590 currently needs that ... - value: guidellm -{% if llmd_run_guidellm_benchmark_run_as_root %} - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: true -{% endif %} - volumeMounts: - - name: home - mountPath: /home/guidellm - - name: results - mountPath: /results - volumes: - - name: home - emptyDir: {} - - name: results - persistentVolumeClaim: - claimName: "{{ llmd_run_guidellm_benchmark_name }}" diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_pvc.yaml.j2 b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_pvc.yaml.j2 deleted file mode 100644 index 192ee989d..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/templates/guidellm_benchmark_pvc.yaml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "{{ llmd_run_guidellm_benchmark_name }}" - namespace: "{{ target_namespace }}" -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ llmd_run_guidellm_benchmark_pvc_size }} \ No newline at end of file diff --git a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/vars/main/resources.yml b/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/vars/main/resources.yml deleted file mode 100644 index 0dfe5ffd3..000000000 --- a/projects/llm_d_legacy/toolbox/llmd_run_guidellm_benchmark/vars/main/resources.yml +++ /dev/null @@ -1,3 +0,0 @@ -guidellm_benchmark_job_template: templates/guidellm_benchmark_job.yaml.j2 -guidellm_benchmark_pvc_template: templates/guidellm_benchmark_pvc.yaml.j2 -copy_helper_pod_template: templates/copy_helper_pod.yaml.j2 diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/defaults/main/config.yml b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/defaults/main/config.yml deleted file mode 100644 index 274d09428..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/defaults/main/config.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Storage.download_to_pvc - -# Parameters -# Name of the data source -# Mandatory value -storage_download_to_pvc_name: - -# URL of the source data -# Mandatory value -storage_download_to_pvc_source: - -# Name of the PVC that will be create to store the dataset files. -# Mandatory value -storage_download_to_pvc_pvc_name: - -# Name of the namespace in which the PVC will be created -# Mandatory value -storage_download_to_pvc_namespace: - -# Path to credentials to use for accessing the dataset. -storage_download_to_pvc_creds: - -# the path where to store the downloaded files, in the PVC -storage_download_to_pvc_storage_dir: / - -# if True, clears the storage directory before downloading. -storage_download_to_pvc_clean_first: false - -# the access mode to request when creating the PVC -storage_download_to_pvc_pvc_access_mode: ReadWriteOnce - -# the size of the PVC to request, when creating the PVC -storage_download_to_pvc_pvc_size: 80Gi - -# the name of the storage class to pass when creating the PVC -storage_download_to_pvc_pvc_storage_class_name: null - -# the image to use for running the download Pod -storage_download_to_pvc_image: registry.access.redhat.com/ubi9/ubi - -# Run the download container as root user -storage_download_to_pvc_run_as_root: false - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/files/entrypoint.sh b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/files/entrypoint.sh deleted file mode 100644 index 5ef876c79..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/files/entrypoint.sh +++ /dev/null @@ -1,180 +0,0 @@ -#! /bin/bash - -set -e -set -u -set -o pipefail -set -x - - -if [[ -z "${STORAGE_DIR:-}" ]]; then - STORAGE_DIR=/storage -fi - -if [[ "$CLEAN_FIRST" == True ]]; then - rm "$STORAGE_DIR" -rf -fi - -mkdir -p "$STORAGE_DIR" -chmod ugo+w "$STORAGE_DIR" || true - -echo "---" - -df -h "$STORAGE_DIR" - -echo "---" - -if [[ -e "$STORAGE_DIR/$SOURCE_NAME" ]]; then - echo "Warning: $STORAGE_DIR/$SOURCE_NAME already exists :/ Cleaning it up." - rm -rfv "$STORAGE_DIR/$SOURCE_NAME" -fi - -if [[ "$DOWNLOAD_SOURCE" == "hf://"* ]]; -then - export HF_HOME=/tmp - export HF_HUB_ENABLE_HF_TRANSFER=1 - export HF_XET_HIGH_PERFORMANCE=1 - export HF_XET_NUM_CONCURRENT_RANGE_GETS=8 - - /storage-initializer/scripts/initializer-entrypoint "$DOWNLOAD_SOURCE" "$STORAGE_DIR/$SOURCE_NAME" -elif [[ "$DOWNLOAD_SOURCE" == "https://huggingface.co/"* ]]; -then - dnf install --quiet -y git-lfs - - if [[ "${CRED_FILE:-}" ]]; - then - echo "Enabling git 'store' credential helper ..." - sha256sum "$CRED_FILE" - - git config --global credential.helper "store --file=$CRED_FILE" - else - echo "No credential file passed." - fi - - if ! time git clone "$DOWNLOAD_SOURCE" "$STORAGE_DIR/$SOURCE_NAME" --depth=1 \ - |& grep -v 'unable to get credential storage lock in 1000 ms: Read-only file system' - then - rm -rf "$STORAGE_DIR/$SOURCE_NAME" - echo "Clone failed :/" - exit 1 - fi - rm -rf "$STORAGE_DIR/$SOURCE_NAME/.git" - -elif [[ "$DOWNLOAD_SOURCE" == "s3://"* ]]; -then - if [[ -z "${CRED_FILE:-}" ]]; - then - echo "ERROR: no credentials provided :/" - exit 1 - fi - if [[ ! -f "$CRED_FILE" ]]; then - echo "ERROR: credentials file does not exist :/" - exit 1 - fi - - dnf install --quiet -y unzip - - echo "Building AWS cli ..." - curl -Ssf "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - ./aws/install - - export AWS_SHARED_CREDENTIALS_FILE=$CRED_FILE - - # check if the source is a directory - echo "Checking if the source exists (aborts silently if the file doesn't exist)" - obj_type=$(aws s3 ls $DOWNLOAD_SOURCE | awk '{print $1}') - - if [[ "$obj_type" == "PRE" ]] - then - echo "Source is a directory. Downloading recursively." - args="--recursive --quiet" - else - echo "Source is a file." - args= - fi - - if ! time aws s3 cp "$DOWNLOAD_SOURCE" "$STORAGE_DIR/$SOURCE_NAME" $args - then - rm -rf "$STORAGE_DIR/$SOURCE_NAME" - echo "Copy failed :/" - exit 1 - fi -elif [[ "$DOWNLOAD_SOURCE" == "dmf://"* ]]; -then - # CRED_FILE --> github token - # CRED_FILE_2 --> DMF JSON token - - if [[ -z "${CRED_FILE:-}" ]]; - then - echo "ERROR: no credentials provided :/" - exit 1 - fi - if [[ ! -f "$CRED_FILE" ]]; then - echo "ERROR: credentials file does not exist :/" - exit 1 - fi - - DMF_VERSION=1.7.1 - DMF_WHL_NAME="dmf_lib-${DMF_VERSION}-py3-none-any.whl" - DMF_WHL_FILE="$STORAGE_DIR/../binaries/$DMF_WHL_NAME" - if [[ ! -f "$DMF_WHL_FILE" ]]; then - echo "ERROR: DMF wheel file does not exist ($DMF_WHL_FILE) :/" - exit 1 - fi - - export PATH=$HOME/.local/bin:$PATH - pip install --quiet "$DMF_WHL_FILE" - - cat > lh-conf.yaml </dev/null 2>&1; then - time find "./$SOURCE_NAME" ! -path '*/.git/*' -type f -exec sha256sum {} \; | tee -a "${SOURCE_NAME}.sha256sum" - echo "---" -fi - -if command -v du >/dev/null 2>&1; then - du -sh "./$SOURCE_NAME" - echo "---" -fi - -if command -v df >/dev/null 2>&1; then - df -h "$STORAGE_DIR" - echo "---" -fi - -exit 0 diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/meta/main.yml b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/meta/main.yml deleted file mode 100644 index 7f867d73b..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -dependencies: diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/tasks/main.yml b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/tasks/main.yml deleted file mode 100644 index 01fba6cdc..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/tasks/main.yml +++ /dev/null @@ -1,163 +0,0 @@ ---- - -- name: Create the src directory - file: - path: "{{ artifact_extra_logs_dir }}/src" - state: directory - mode: '0755' - -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' - -- name: Store the safe name of the data source - set_fact: - data_source_name: "{{ storage_download_to_pvc_name | replace('.', '-') | replace('_', '-') }}" - -- name: Store the name of the objects - set_fact: - download_name: download-{{ data_source_name | lower }}-to-{{ storage_download_to_pvc_pvc_name | lower }} - -- name: Delete the creds secret, if it exists - command: - oc delete secret/{{ download_name }} - -n {{ storage_download_to_pvc_namespace }} - --ignore-not-found=true - when: storage_download_to_pvc_creds | length > 0 - -- name: Create the s3 secret - command: - oc create secret generic {{ download_name }} - -n {{ storage_download_to_pvc_namespace }} - --from-file=cred={{ storage_download_to_pvc_creds }} - when: storage_download_to_pvc_creds | length > 0 - -- name: Create the entrypoint ConfigMap file - shell: - oc create cm {{ download_name }}-entrypoint - --from-file="entrypoint.sh={{ entrypoint }}" - -n {{ storage_download_to_pvc_namespace }} - --dry-run=client - -oyaml - > {{ artifact_extra_logs_dir }}/src/configmap_entrypoint.yaml - -- name: Create the entrypoint ConfigMap resource - command: - oc apply -f {{ artifact_extra_logs_dir }}/src/configmap_entrypoint.yaml - -- name: Apply the PVC template - template: - src: "{{ pvc_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/pvc.yaml" - mode: '0400' - -- name: Check if the PVC already exists - command: oc get -f "{{ artifact_extra_logs_dir }}/src/pvc.yaml" -oname - failed_when: false - register: has_pvc_cmd - -- name: Create a PVC if it does not exist - when: has_pvc_cmd.rc != 0 - command: - oc apply -f "{{ artifact_extra_logs_dir }}/src/pvc.yaml" - -- name: Check if the PVC has the right access mode - when: has_pvc_cmd.rc == 0 - command: - oc get -f "{{ artifact_extra_logs_dir }}/src/pvc.yaml" - -ojsonpath={.spec.accessModes} - register: pvc_access_mode_cmd - -- name: Show a message if the PVC access mode isn't the expeced one - debug: msg="Unexpected access mode in the PVC. Found '{{ pvc_access_mode_cmd.stdout }}'. Expected '{{ storage_download_to_pvc_pvc_access_mode }}' " - when: - - has_pvc_cmd.rc == 0 - - storage_download_to_pvc_pvc_access_mode not in pvc_access_mode_cmd.stdout - -- name: Apply the Pod template - template: - src: "{{ pod_template }}" - dest: "{{ artifact_extra_logs_dir }}/src/pod.yaml" - mode: '0400' - -- name: Delete the Pod, if it exists - command: - oc delete -f "{{ artifact_extra_logs_dir }}/src/pod.yaml" - --ignore-not-found=true - -- name: Create a Pod for downloading the dataset - command: - oc apply -f "{{ artifact_extra_logs_dir }}/src/pod.yaml" - -- name: Wait for the download and delete the secret - block: - - name: Wait for the downloader Pod to start - shell: - set -o pipefail; - - oc get -f "{{ artifact_extra_logs_dir }}/src/pod.yaml" - --no-headers - | awk '{print $3}' - register: wait_downloader_pod_start_status - until: "'Pending' not in wait_downloader_pod_start_status.stdout " - retries: 30 - delay: 10 - - - name: Wait for the downloader Pod to terminate - shell: - set -o pipefail; - - oc get -f "{{ artifact_extra_logs_dir }}/src/pod.yaml" - --no-headers - | awk '{print $3}' - - register: wait_downloader_pod_cmd - until: "'Completed' in wait_downloader_pod_cmd.stdout - or 'Failed' in wait_downloader_pod_cmd.stdout - or 'Error' in wait_downloader_pod_cmd.stdout - or 'ErrImagePull' in wait_downloader_pod_cmd.stdout - or 'ImagePullBackOff' in wait_downloader_pod_cmd.stdout - or not wait_downloader_pod_cmd.stdout" - retries: 900 - delay: 20 - always: - - name: Delete the creds secret, if it was created - command: - oc delete secret/{{ download_name }} - -n {{ storage_download_to_pvc_namespace }} - --ignore-not-found=true - when: storage_download_to_pvc_creds | length > 0 - - - name: Get the logs of the download Pod (debug) - shell: - oc logs {{ download_name }} - -n {{ storage_download_to_pvc_namespace }} - > {{ artifact_extra_logs_dir }}/artifacts/pod.log - ignore_errors: true - - - name: Get the description of the download Pod (debug) - shell: - oc describe pod/{{ download_name }} - -n {{ storage_download_to_pvc_namespace }} - > {{ artifact_extra_logs_dir }}/artifacts/pod.desc - ignore_errors: true - - - name: Capture the state of the PVC - shell: - oc get -f "{{ artifact_extra_logs_dir }}/src/pvc.yaml" - -oyaml > "{{ artifact_extra_logs_dir }}/artifacts/pvc.yaml" - ignore_errors: true - -- name: Label the PVC if the download succeeded - when: "'Completed' in wait_downloader_pod_cmd.stdout" - command: - oc label pvc/{{ storage_download_to_pvc_pvc_name }} - -n {{ storage_download_to_pvc_namespace }} - {{ storage_download_to_pvc_name }}=yes - --overwrite - -- name: Fail if the the downloader Pod failed - when: "'Completed' not in wait_downloader_pod_cmd.stdout" - fail: msg="The download failed ({{ wait_downloader_pod_cmd.stdout }})..." diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pod.yml.j2 b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pod.yml.j2 deleted file mode 100644 index cf377c657..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pod.yml.j2 +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: {{ download_name }} - namespace: {{ storage_download_to_pvc_namespace }} - labels: - download-to-pvc: "yes" -spec: - restartPolicy: Never - containers: - - name: downloader - image: {{ storage_download_to_pvc_image }} -{% if storage_download_to_pvc_run_as_root %} - command: [bash, -c, "/mnt/entrypoint/entrypoint.sh && chmod -R a+rX /storage"] - securityContext: - runAsUser: 0 - allowPrivilegeEscalation: true -{% else %} - command: [bash, /mnt/entrypoint/entrypoint.sh] -{% endif %} - env: - - name: DOWNLOAD_SOURCE - value: "{{ storage_download_to_pvc_source }}" -{% if storage_download_to_pvc_creds | length %} - - name: CRED_FILE - value: /mnt/download-secret/cred -{% endif %} - - name: SOURCE_NAME - value: {{ storage_download_to_pvc_name }} - - name: STORAGE_DIR - value: /storage{{ storage_download_to_pvc_storage_dir if storage_download_to_pvc_storage_dir.startswith('/') else '/' + storage_download_to_pvc_storage_dir }} - - name: CLEAN_FIRST - value: "{{ storage_download_to_pvc_clean_first }}" - volumeMounts: -{% if storage_download_to_pvc_creds | length %} - - name: download-secret - mountPath: /mnt/download-secret - readOnly: true -{% endif %} - - name: entrypoint-cm - mountPath: /mnt/entrypoint/entrypoint.sh - readOnly: true - subPath: entrypoint.sh - - name: storage - mountPath: /storage - volumes: -{% if storage_download_to_pvc_creds | length %} - - name: download-secret - secret: - secretName: {{ download_name }} -{% endif %} - - name: storage - persistentVolumeClaim: - claimName: {{ storage_download_to_pvc_pvc_name }} - - name: entrypoint-cm - configMap: - defaultMode: 0777 - name: {{ download_name }}-entrypoint diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pvc.yml.j2 b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pvc.yml.j2 deleted file mode 100644 index 2e0a3b816..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/templates/pvc.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ storage_download_to_pvc_pvc_name }} - namespace: {{ storage_download_to_pvc_namespace }} -spec: - accessModes: - - {{ storage_download_to_pvc_pvc_access_mode }} - resources: - requests: - storage: {{ storage_download_to_pvc_pvc_size }} -{% if storage_download_to_pvc_pvc_storage_class_name %} - storageClassName: {{ storage_download_to_pvc_pvc_storage_class_name }} -{% endif %} diff --git a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/vars/main/resources.yml b/projects/llm_d_legacy/toolbox/storage_download_to_pvc/vars/main/resources.yml deleted file mode 100644 index 5285dcd2c..000000000 --- a/projects/llm_d_legacy/toolbox/storage_download_to_pvc/vars/main/resources.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -entrypoint: '{{ role_path }}/files/entrypoint.sh' -pod_template: 'templates/pod.yml.j2' -pvc_template: 'templates/pvc.yml.j2' diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/analyze/__init__.py b/projects/llm_d_legacy/visualizations/llmd_inference/analyze/__init__.py deleted file mode 100644 index 8fb158d04..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/analyze/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# Analysis functions for LLM-D inference results - - -def analyze_guidellm_performance(results): - """ - Analyze Guidellm benchmark performance and provide insights - """ - if not results.guidellm_benchmarks: - return "No Guidellm benchmark data available" - - insights = [] - benchmarks = results.guidellm_benchmarks - - # Find best performing strategy - best_strategy = max(benchmarks, key=lambda x: x.request_rate) - insights.append( - f"✓ Best strategy: {best_strategy.strategy} ({best_strategy.request_rate:.2f} req/s)" - ) - - # Analyze scaling behavior - constant_strategies = [b for b in benchmarks if "constant" in b.strategy] - if len(constant_strategies) > 2: - rates = [b.request_rate for b in constant_strategies] - min_rate = min(rates) - if min_rate > 0 and max(rates) / min_rate > 3: - insights.append("✓ Good throughput scaling with concurrency") - elif min_rate == 0: - insights.append("⚠ Cannot analyze scaling (zero request rate detected)") - else: - insights.append("⚠ Limited throughput scaling") - - # Analyze latency consistency - latencies = [b.request_latency_median for b in benchmarks] - min_latency = min(latencies) - if min_latency > 0 and max(latencies) / min_latency > 10: - insights.append("⚠ High latency variance across strategies") - elif min_latency == 0: - insights.append("⚠ Cannot analyze latency consistency (zero latency detected)") - else: - insights.append("✓ Consistent latency across strategies") - - return "\n".join(insights) - - -def analyze_overall_performance(results, kpis): - """ - Provide overall performance analysis and recommendations - """ - insights = [] - - # Overall test success - if results.test_success: - insights.append("✓ Test completed successfully") - else: - insights.append(f"✗ Test failed: {results.test_failure_reason}") - - # Performance score analysis - if "overall_performance_score" in kpis and kpis["overall_performance_score"]: - score = kpis["overall_performance_score"] - if score > 0.8: - insights.append(f"✓ Excellent performance score: {score:.3f}") - elif score > 0.6: - insights.append(f"✓ Good performance score: {score:.3f}") - else: - insights.append(f"⚠ Performance needs improvement: {score:.3f}") - - # Recommendations - recommendations = [] - - if results.guidellm_benchmarks: - max_rate = max(b.request_rate for b in results.guidellm_benchmarks) - if max_rate < 1.0: - recommendations.append("• Consider scaling up inference resources") - - if recommendations: - insights.extend(["", "Recommendations:"] + recommendations) - - return "\n".join(insights) diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/data/plots.yaml b/projects/llm_d_legacy/visualizations/llmd_inference/data/plots.yaml deleted file mode 100644 index 056e54886..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/data/plots.yaml +++ /dev/null @@ -1,13 +0,0 @@ -visualize: -- id: llmd_inference - generate: - - "report: LLM-D Inference Error Report" - - "report: GuideLLM Results Table" - - "report: GuideLLM Performance Analysis" - - "report: Prometheus Resource Usage" - - "report: Prometheus GPU Performance" - - "report: Prometheus System Health" - - "report: VLLM Metrics Analysis" - - "report: Baseline Comparisons" - - "report: Intelligent-routing Comparisons" - - "report: P-D Comparisons" diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/models/__init__.py b/projects/llm_d_legacy/visualizations/llmd_inference/models/__init__.py deleted file mode 100644 index b3b945e60..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .lts import * -from .kpi import * diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/models/kpi.py b/projects/llm_d_legacy/visualizations/llmd_inference/models/kpi.py deleted file mode 100644 index 5b0ddeacf..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/models/kpi.py +++ /dev/null @@ -1,3 +0,0 @@ -import matrix_benchmarking.models as matbench_models - -KPIs = {} # populated by the @matbench_models.KPIMetadata decorator diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/models/lts.py b/projects/llm_d_legacy/visualizations/llmd_inference/models/lts.py deleted file mode 100644 index 1fb2ee0b1..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/models/lts.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -from typing import Any, List, Optional, Dict - -from pydantic import BaseModel, Field - -import matrix_benchmarking.models as matbench_models -from . import kpi - -KPI_SETTINGS_VERSION = "1.0" - - -class Settings(BaseModel): - kpi_settings_version: str = KPI_SETTINGS_VERSION - - # Allow any extra fields without validation - class Config: - extra = "allow" - - -class GuidellmBenchmark(matbench_models.ExclusiveModel): - strategy: str - duration: float - warmup_time: float - cooldown_time: float - - # Request metrics - request_rate: float - request_concurrency: float - completed_requests: int - failed_requests: int - - # Token metrics (per request) - input_tokens_per_request: float - output_tokens_per_request: float - total_tokens_per_request: float - - # Latency metrics - request_latency_median: float - request_latency_p95: float - ttft_median: float - ttft_p95: float - itl_median: float # Inter Token Latency - itl_p95: float - tpot_median: float # Time Per Output Token - tpot_p95: float - - # TTFT percentiles - ttft_p10: float = 0.0 - ttft_p25: float = 0.0 - ttft_p50: float = 0.0 - ttft_p75: float = 0.0 - ttft_p90: float = 0.0 - - # ITL percentiles - itl_p10: float = 0.0 - itl_p25: float = 0.0 - itl_p50: float = 0.0 - itl_p75: float = 0.0 - itl_p90: float = 0.0 - - # Throughput metrics - tokens_per_second: float - input_tokens_per_second: float - output_tokens_per_second: float - - # Output token percentiles - output_tokens_per_second_p10: float = 0.0 - output_tokens_per_second_p25: float = 0.0 - output_tokens_per_second_p50: float = 0.0 - output_tokens_per_second_p75: float = 0.0 - output_tokens_per_second_p90: float = 0.0 - - -LTS_SCHEMA_VERSION = "1.0" - - -class Metadata(matbench_models.Metadata): - lts_schema_version: str - settings: Settings - presets: List[str] = Field(default_factory=list) - - -class Results(matbench_models.ExclusiveModel): - # Test metadata - test_name: str - test_success: bool = True - test_failure_reason: Optional[str] = None - - # Benchmark results - guidellm_benchmarks: List[GuidellmBenchmark] = Field(default_factory=list) - guidellm_configuration: Optional[dict] = None - guidellm_log_path: Optional[str] = None - prometheus_path: Optional[str] = None - - # LLMISVC deployment configuration - llmisvc_config: Optional[dict] = None - - -class LlmdInferenceKPI(matbench_models.KPI, Settings): - pass - - -LlmdInferenceKPIs = matbench_models.getKPIsModel( - "LlmdInferenceKPIs", __name__, kpi.KPIs, LlmdInferenceKPI -) - - -class Payload(matbench_models.ExclusiveModel): - metadata: Metadata - results: Results - kpis: Optional[LlmdInferenceKPIs] = None diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/__init__.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/__init__.py deleted file mode 100644 index 6e24757b6..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Register all plotting modules -def register(): - """Register all plotting functions with matrix benchmarking framework""" - - # Import and register each module - from . import report - from . import error_report - from . import throughput_analysis - from . import throughput_comparisons - from . import prometheus - from . import vllm_metrics - - report.register() - error_report.register() - throughput_analysis.register() - throughput_comparisons.register() - prometheus.register() - vllm_metrics.register() - vllm_metrics.register_report() diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/error_report.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/error_report.py deleted file mode 100644 index 302aa3a2b..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/error_report.py +++ /dev/null @@ -1,629 +0,0 @@ -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common -from dash import html -import yaml -import pathlib -import copy - - -def analyze_llmisvc_deployment(llmisvc_config): - """ - Analyze LLMISVC deployment configuration - - Args: - llmisvc_config: Parsed LLMISVC configuration dict - - Returns formatted HTML content showing deployment details - """ - if not llmisvc_config: - return [html.P("❌ No LLMISVC configuration available")] - - isvc_data = llmisvc_config - - def safe_get(data, path, default="missing"): - """Safely navigate nested dict path""" - try: - current = data - for key in path.split("."): - if "[" in key and "]" in key: - # Handle array access like containers[0] - array_key, index = key.split("[") - index = int(index.rstrip("]")) - current = current[array_key][index] - else: - current = current[key] - return current - except (KeyError, IndexError, TypeError): - return default - - def get_vllm_args(containers_section): - """Extract VLLM_ADDITIONAL_ARGS from containers section""" - if containers_section == "missing": - return "missing" - try: - for container in containers_section: - if container.get("name") == "main": - env_vars = container.get("env", []) - for env_var in env_vars: - if env_var.get("name") == "VLLM_ADDITIONAL_ARGS": - return env_var.get("value", "empty") - return "not set" - except (TypeError, KeyError): - return "missing" - - def get_epp_config(router_args): - """Extract EPP config from router scheduler args""" - if router_args == "missing": - return "router not defined" - try: - if isinstance(router_args, list) and len(router_args) > 0: - # Find --config-text and get the next argument - for i, arg in enumerate(router_args): - if arg == "--config-text" and i + 1 < len(router_args): - return router_args[i + 1] - return "not set" - except (TypeError, IndexError): - return "missing" - - def format_resources(resources): - """Format resource requirements for display""" - if resources == "missing": - return "missing" - try: - return "\n" + yaml.dump(dict(resources=resources)).replace("resources:\n", "") - except (TypeError, AttributeError): - return "invalid format" - - # Build analysis content - analysis_content = [] - - # Model spec - model_uri = safe_get(isvc_data, "spec.model.uri") - model_name = safe_get(isvc_data, "spec.model.name") - - # Main template - main_replicas = safe_get(isvc_data, "spec.replicas") - main_containers = safe_get(isvc_data, "spec.template.containers") - main_vllm_args = get_vllm_args(main_containers) - main_resources = safe_get(isvc_data, "spec.template.containers[0].resources") - - # Router/EPP - router_args = safe_get(isvc_data, "spec.router.scheduler.template.containers[0].args") - epp_config = get_epp_config(router_args) - - # Prefill (if exists) - prefill_replicas = safe_get(isvc_data, "spec.prefill.replicas") - prefill_containers = ( - safe_get(isvc_data, "spec.prefill.template.containers") - if prefill_replicas != "missing" - else "missing" - ) - prefill_vllm_args = ( - get_vllm_args(prefill_containers) if prefill_replicas != "missing" else "missing" - ) - prefill_resources = ( - safe_get(isvc_data, "spec.prefill.template.containers[0].resources") - if prefill_replicas != "missing" - else "missing" - ) - - # Code block styling - code_style = { - "background-color": "#f8f8f8", - "padding": "10px", - "border-radius": "5px", - "font-family": "monospace", - "font-size": "0.9em", - "white-space": "pre-wrap", - "overflow": "auto", - "margin": "10px 0", - } - - # Model specification section - analysis_content.append(html.H6("=== MODEL SPECIFICATION ===")) - model_lines = [ - f"name: {model_name}", - f"uri: {model_uri}", - ] - analysis_content.append(html.Pre("\n".join(model_lines), style=code_style)) - - # Main template section - analysis_content.append(html.H6("=== MAIN TEMPLATE ===")) - main_lines = [ - f"replicas: {main_replicas}", - "", - f"VLLM_ADDITIONAL_ARGS: \n- " + f"\n- ".join(main_vllm_args.split()), - "", - f"resources: {format_resources(main_resources)}", - ] - analysis_content.append(html.Pre("\n".join(main_lines), style=code_style)) - - # Router/EPP configuration section - analysis_content.append(html.H6("=== ROUTER/EPP CONFIGURATION ===")) - if epp_config not in ["missing", "not set"]: - epp_lines = [epp_config] - else: - epp_lines = [f"EPP Config: {epp_config}"] - analysis_content.append(html.Pre("\n".join(epp_lines), style=code_style)) - - # Prefill template section - analysis_content.append(html.H6("=== PREFILL TEMPLATE ===")) - if prefill_replicas != "missing": - prefill_lines = [ - f"replicas: {prefill_replicas}", - "", - f"VLLM_ADDITIONAL_ARGS: \n-" + "\n- ".join(prefill_vllm_args.split()), - "", - f"resources: {format_resources(prefill_resources)}", - ] - else: - prefill_lines = ["prefill not defined"] - - analysis_content.append(html.Pre("\n".join(prefill_lines), style=code_style)) - - return analysis_content - - -def analyze_guidellm_configuration(results): - """ - Analyze GuideLLM benchmark configuration - - Args: - results: Results object containing guidellm_benchmarks and potentially args - - Returns formatted HTML content showing benchmark configuration - """ - if not hasattr(results, "guidellm_benchmarks") or not results.guidellm_benchmarks: - return [html.P("❌ No GuideLLM benchmark data available")] - - analysis_content = [] - - # Code block styling - code_style = { - "background-color": "#f8f8f8", - "padding": "10px", - "border-radius": "5px", - "font-family": "monospace", - "font-size": "0.9em", - "white-space": "pre-wrap", - "overflow": "auto", - "margin": "10px 0", - } - - # Top-level configuration (including args) - analysis_content.append(html.H6("=== GUIDELLM CONFIGURATION ===")) - - config_lines = [] - - # Show configuration if available - if hasattr(results, "guidellm_configuration") and results.guidellm_configuration: - cfg = copy.deepcopy(results.guidellm_configuration) - for key in results.guidellm_configuration.get("args", {}).keys(): - if cfg["args"][key] in (None, [], {}): - del cfg["args"][key] - - config_lines.append(yaml.dump(cfg)) - - analysis_content.append(html.Pre("\n".join(config_lines), style=code_style)) - - return analysis_content - - -def register(): - LlmdInferenceErrorReport() - - -class LlmdInferenceErrorReport: - def __init__(self): - self.name = "report: LLM-D Inference Error Report" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate error analysis report showing what has been tested - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("LLM-D Inference Test Analysis")) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Count successful vs failed tests - total_tests = len(entries) - successful_tests = 0 - failed_tests = 0 - guidellm_available = 0 - prometheus_available = 0 - llmisvc_available = 0 - - for entry in entries: - results = entry.results - - # Count test outcomes - if hasattr(results, "test_success") and results.test_success: - successful_tests += 1 - else: - failed_tests += 1 - - # Count available data sources - if hasattr(results, "guidellm_benchmarks") and results.guidellm_benchmarks: - guidellm_available += 1 - - if hasattr(results, "metrics") and results.metrics: - prometheus_available += 1 - - if hasattr(results, "llmisvc_config") and results.llmisvc_config: - llmisvc_available += 1 - - # Overall test summary - header.append(html.H3("Test Summary")) - header.append(html.P(f"Total tests executed: {total_tests}")) - - if successful_tests > 0: - header.append( - html.P( - [ - "✅ Successful tests: ", - html.Span( - str(successful_tests), style={"color": "green", "font-weight": "bold"} - ), - f" ({successful_tests / total_tests * 100:.1f}%)", - ] - ) - ) - - if failed_tests > 0: - header.append( - html.P( - [ - "❌ Failed tests: ", - html.Span(str(failed_tests), style={"color": "red", "font-weight": "bold"}), - f" ({failed_tests / total_tests * 100:.1f}%)", - ] - ) - ) - - header.append(html.Br()) - - # Data availability summary - header.append(html.H3("Data Sources Available")) - header.append( - html.Ul( - [ - html.Li( - [ - "🎯 GuideLLM benchmarks: ", - html.Span( - f"{guidellm_available}/{total_tests}", style={"font-weight": "bold"} - ), - f" ({guidellm_available / total_tests * 100:.1f}%)", - ] - ), - html.Li( - [ - "📊 Prometheus monitoring: ", - html.Span( - f"{prometheus_available}/{total_tests}", - style={"font-weight": "bold"}, - ), - f" ({prometheus_available / total_tests * 100:.1f}%)", - ] - ), - html.Li( - [ - "🚀 LLMISVC configuration: ", - html.Span( - f"{llmisvc_available}/{total_tests}", style={"font-weight": "bold"} - ), - f" ({llmisvc_available / total_tests * 100:.1f}%)", - ] - ), - ] - ) - ) - - header.append(html.Br()) - - # Detailed test breakdown - header.append(html.H3("Individual Test Details")) - - for i, entry in enumerate(entries): - results = entry.results - test_status = ( - "✅ PASSED" - if (hasattr(results, "test_success") and results.test_success) - else "❌ FAILED" - ) - - header.append(html.H4(f"Test {i + 1}: {test_status}")) - - # Test metadata if available - if hasattr(results, "test_name"): - header.append(html.P(f"Test name: {results.test_name}")) - - # Show test entry labels - labels_list = [] - - for key, value in entry.settings.__dict__.items(): - labels_list.append( - html.Code( - f"{key}={value}", - style={ - "background-color": "#f8f8f8", - "padding": "2px 4px", - "margin": "2px", - "border-radius": "3px", - "font-size": "0.9em", - }, - ) - ) - labels_list.append(" ") - - header.append(html.P(["Settings: ", html.Span(labels_list)])) - - header.append( - html.P( - [ - "Directory: ", - html.Code( - str(entry.location).strip("./"), - style={ - "background-color": "#f8f8f8", - "padding": "2px 4px", - "border-radius": "3px", - "font-size": "0.9em", - }, - ), - ] - ) - ) - - if hasattr(results, "test_failure_reason") and results.test_failure_reason: - header.append( - html.P( - [ - "Failure reason: ", - html.Span( - results.test_failure_reason, - style={"color": "red", "font-style": "italic"}, - ), - ] - ) - ) - - # Get artifacts base directory for links - artifacts_basedir = getattr(results, "from_local_env", {}) - if hasattr(artifacts_basedir, "artifacts_basedir"): - artifacts_basedir = artifacts_basedir.artifacts_basedir - else: - artifacts_basedir = None - - # Create links to key artifacts - artifact_links = [] - - if artifacts_basedir: - # Main test artifacts directory - artifact_links.append( - html.Li( - [ - html.A( - "🗂️ Test artifacts directory", - href=str(artifacts_basedir), - target="_blank", - ) - ] - ) - ) - - # LLM Inference Service deployment logs - llm_deployment_dir = artifacts_basedir / "001__deploy_llm_inference_service" - if llm_deployment_dir.exists(): - artifact_links.append( - html.Li( - [ - html.A( - "🚀 LLM Inference Service deployment", - href=str(llm_deployment_dir), - target="_blank", - ) - ] - ) - ) - - # GuideLLM benchmark results - if hasattr(results, "guidellm_benchmarks") and results.guidellm_benchmarks: - strategy_count = len(results.guidellm_benchmarks) - - # Look for guidellm benchmark directory - guidellm_dir = None - for item in artifacts_basedir.iterdir(): - if item.is_dir() and "guidellm_benchmark" in item.name: - guidellm_dir = item - break - - if guidellm_dir: - artifact_links.append( - html.Li( - [ - html.A( - f"📊 GuideLLM benchmark results ({strategy_count} strategies)", - href=str(guidellm_dir), - target="_blank", - ) - ] - ) - ) - - # Link to specific log file if it exists - log_file = ( - guidellm_dir - / "000__llmd__run_guidellm_benchmark" - / "artifacts" - / "guidellm_benchmark_job.logs" - ) - if log_file.exists(): - artifact_links.append( - html.Li( - [ - " └─ ", - html.A( - "benchmark logs", href=str(log_file), target="_blank" - ), - ] - ) - ) - - # Prometheus monitoring data - if hasattr(results, "metrics") and results.metrics: - # Look for prometheus dump directory - for prom_name in results.metrics: - if results.metrics[prom_name]: - artifact_links.append( - html.Li( - [ - f"📈 Prometheus '{prom_name}' monitoring data", - ] - ) - ) - - # Add PROW artifacts link if available - if hasattr(results, "from_env") and results.from_env: - if hasattr(results.from_env, "test") and hasattr(results.from_env.test, "urls"): - prow_artifacts = results.from_env.test.urls.get("PROW_ARTIFACTS") - if prow_artifacts: - artifact_links.append( - html.Li( - [html.A("🔗 PROW Artifacts", href=prow_artifacts, target="_blank")] - ) - ) - - # Show artifact links if available - if artifact_links: - header.append(html.P("🔗 Artifact links:")) - header.append(html.Ul(artifact_links)) - else: - header.append(html.P("🔗 Artifact links: Not available")) - - # Data sources status summary - data_sources = [] - if hasattr(results, "guidellm_benchmarks") and results.guidellm_benchmarks: - strategy_count = len(results.guidellm_benchmarks) - data_sources.append(f"✅ GuideLLM ({strategy_count} strategies)") - else: - data_sources.append("❌ GuideLLM") - - if hasattr(results, "metrics") and results.metrics: - data_sources.append("✅ Prometheus") - else: - data_sources.append("❌ Prometheus") - - header.append(html.P("Data sources: " + " | ".join(data_sources))) - - # LLMISVC deployment analysis - header.append(html.H5("LLMISVC Deployment Configuration")) - llmisvc_analysis = analyze_llmisvc_deployment(results.llmisvc_config) - header.extend(llmisvc_analysis) - - header.append(html.Br()) - - # GuideLLM benchmark analysis - header.append(html.H5("GuideLLM Benchmark Configuration")) - guidellm_analysis = analyze_guidellm_configuration(results) - header.extend(guidellm_analysis) - - header.append(html.Br()) - - # Summary section with PROW artifacts links - header.append(html.H3("PROW Artifacts Summary")) - header.append( - html.P( - "Quick access to PROW artifacts for all test entries organized by platform → model → load_shape:" - ) - ) - - # Filter out unwanted settings keys - FILTERED_SETTINGS_KEYS = {"owner", "llm-d", "platform", "model", "load_shape", "version"} - - # Group entries hierarchically - hierarchy = {} - for i, entry in enumerate(entries): - results = entry.results - - # Get hierarchy values - platform = getattr(entry.settings, "platform", "unknown") - version = getattr(entry.settings, "version", "") - # Concatenate version to platform name - platform_with_version = f"{platform}-{version}" if version else platform - - model = getattr(entry.settings, "model", "unknown") - load_shape = getattr(entry.settings, "load_shape", "unknown") - - # Initialize hierarchy structure - if platform_with_version not in hierarchy: - hierarchy[platform_with_version] = {} - if model not in hierarchy[platform_with_version]: - hierarchy[platform_with_version][model] = {} - if load_shape not in hierarchy[platform_with_version][model]: - hierarchy[platform_with_version][model][load_shape] = [] - - # Build filtered settings string - settings_parts = [] - for key, value in entry.settings.__dict__.items(): - if key not in FILTERED_SETTINGS_KEYS: - settings_parts.append(f"{key}={value}") - settings_str = ", ".join(settings_parts) - - # Get PROW artifacts URL - prow_url = None - if hasattr(results, "from_env") and results.from_env: - if hasattr(results.from_env, "test") and hasattr(results.from_env.test, "urls"): - prow_url = results.from_env.test.urls.get("PROW_ARTIFACTS") - - # Add to hierarchy - hierarchy[platform_with_version][model][load_shape].append( - {"settings_str": settings_str, "prow_url": prow_url} - ) - - # Render hierarchical structure - for platform_version in sorted(hierarchy.keys()): - header.append(html.H2(f"🖥️ {platform_version}")) - - for model in sorted(hierarchy[platform_version].keys()): - header.append(html.H3(f" 🤖 Model: {model}")) - - for load_shape in sorted(hierarchy[platform_version][model].keys()): - header.append(html.H4(f" 📊 {load_shape} load shape")) - - load_shape_list = [] - for entry_data in hierarchy[platform_version][model][load_shape]: - if entry_data["prow_url"]: - list_item_content = [ - html.A( - entry_data["settings_str"], - href=entry_data["prow_url"], - target="_blank", - style={"color": "#0066cc", "text-decoration": "underline"}, - ) - ] - else: - list_item_content = [ - f"{entry_data['settings_str']} (No PROW artifacts available)" - ] - - load_shape_list.append(html.Li(list_item_content)) - - header.append(html.Ul(load_shape_list, style={"margin-left": "20px"})) - - header.append(html.Br()) - - return None, header diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus.py deleted file mode 100644 index 4cd98ef52..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus.py +++ /dev/null @@ -1,12 +0,0 @@ -import projects.matrix_benchmarking.visualizations.helpers.store.prom as helpers_store_prom -import projects - -llmd = getattr(projects, "llm_d_legacy") - - -def register(): - """ - Register Prometheus metrics plots for LLM-D inference visualization - """ - get_llmd_main_metrics = llmd.visualizations.llmd_inference.store.parsers.get_llmd_main_metrics - get_llmd_main_metrics(register=True) diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus_reports.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus_reports.py deleted file mode 100644 index dbc44a00c..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/prometheus_reports.py +++ /dev/null @@ -1,244 +0,0 @@ -from dash import html - -import projects.matrix_benchmarking.visualizations.helpers.plotting.report as report - -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common - -from .utils import check_prometheus_data_availability - - -def register(): - """Register all Prometheus report classes""" - PrometheusResourceUsageReport() - PrometheusGPUPerformanceReport() - PrometheusSystemHealthReport() - - -class PrometheusResourceUsageReport: - def __init__(self): - self.name = "report: Prometheus Resource Usage" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate comprehensive resource usage report from Prometheus metrics - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("📊 Resource Usage Analytics")) - header.append( - html.P("Comprehensive CPU and memory usage analysis for LLM-D inference workloads") - ) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Check if main Prometheus metrics data is available - has_data, error_elements = check_prometheus_data_availability( - "sutest", "resource usage report" - ) - if not has_data: - header.extend(error_elements) - return None, header - - # Application Resource Usage Section - header.append(html.H3("🚀 LLM-D Application Resources")) - header.append(html.P("CPU and memory usage for LLM-D inference services")) - - header += report.Plot_and_Text("Prom: LLM Inference Service: CPU usage", args) - header += report.Plot_and_Text("Prom: LLM Inference Service: Mem usage", args) - header += report.Plot_and_Text("Prom: LLM Inference Gateway: CPU usage", args) - header += report.Plot_and_Text("Prom: LLM Inference Gateway: Mem usage", args) - - header.append(html.Br()) - - # Cluster Resource Usage Section - header.append(html.H3("🏗️ Cluster Resource Overview")) - header.append(html.P("Overall cluster CPU and memory utilization")) - - header += report.Plot_and_Text("Prom: sutest cluster CPU usage", args) - header += report.Plot_and_Text("Prom: sutest cluster memory usage", args) - - header.append(html.Br()) - - # Node-level Resource Usage Section - header.append(html.H3("🖥️ Node-level Resource Usage")) - header.append(html.P("CPU usage and idle time breakdown by node type")) - - header += report.Plot_and_Text("Prom: Sutest Control Plane Node CPU usage", args) - header += report.Plot_and_Text("Prom: Sutest Worker Node CPU usage", args) - header += report.Plot_and_Text("Prom: Sutest Control Plane Node CPU idle", args) - header += report.Plot_and_Text("Prom: Sutest Worker Node CPU idle", args) - - return None, header - - -class PrometheusGPUPerformanceReport: - def __init__(self): - self.name = "report: Prometheus GPU Performance" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate comprehensive GPU performance report from Prometheus DCGM metrics - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("🎮 GPU Performance Analytics")) - header.append( - html.P( - "Comprehensive GPU utilization, memory, and throughput analysis using DCGM metrics" - ) - ) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Check if main Prometheus metrics data is available - has_data, error_elements = check_prometheus_data_availability( - "sutest", "GPU performance report" - ) - if not has_data: - header.extend(error_elements) - return None, header - - # GPU Memory Usage Section - header.append(html.H3("💾 GPU Memory Utilization")) - header.append(html.P("GPU memory consumption and allocation patterns")) - - header += report.Plot_and_Text("Prom: Sutest GPU memory used", args) - header += report.Plot_and_Text("Prom: Sutest GPU memory used (all GPUs)", args) - header += report.Plot_and_Text("Prom: Sutest GPU memory unallocated", args) - header += report.Plot_and_Text("Prom: Sutest GPU memory transfer utilization", args) - - header.append(html.Br()) - - # GPU Compute Performance Section - header.append(html.H3("⚡ GPU Compute Performance")) - header.append(html.P("GPU compute utilization and active processing units")) - - header += report.Plot_and_Text( - "Prom: Sutest GPU compute utilization (not 100% accurate)", args - ) - header += report.Plot_and_Text("Prom: Sutest GPU engine usage (not 100% accurate)", args) - header += report.Plot_and_Text("Prom: Sutest GPU active computes", args) - header += report.Plot_and_Text("Prom: Sutest GPU computes occupancy", args) - - header.append(html.Br()) - - # GPU Pipeline Usage Section - header.append(html.H3("🔧 GPU Pipeline Utilization")) - header.append(html.P("GPU floating-point pipeline usage by precision")) - - header += report.Plot_and_Text("Prom: Sutest GPU active fp16 pipe", args) - header += report.Plot_and_Text("Prom: Sutest GPU active fp32 pipe", args) - header += report.Plot_and_Text("Prom: Sutest GPU active fp64 pipe", args) - - header.append(html.Br()) - - # GPU Interconnect Performance Section - header.append(html.H3("🔗 GPU Interconnect Performance")) - header.append(html.P("NVLink and PCIe transfer rates and throughput")) - - header += report.Plot_and_Text("Prom: Sutest GPU NVLink transfer (rx)", args) - header += report.Plot_and_Text("Prom: Sutest GPU NVLink transfer (tx)", args) - header += report.Plot_and_Text("Prom: Sutest GPU PCIe transfer (rx)", args) - header += report.Plot_and_Text("Prom: Sutest GPU PCIe transfer (tx)", args) - - return None, header - - -class PrometheusSystemHealthReport: - def __init__(self): - self.name = "report: Prometheus System Health" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate comprehensive system health report from Prometheus cluster metrics - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("🏥 System Health Analytics")) - header.append( - html.P("Kubernetes cluster health monitoring including API server and ETCD performance") - ) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Check if main Prometheus metrics data is available - has_data, error_elements = check_prometheus_data_availability( - "sutest", "system health report" - ) - if not has_data: - header.extend(error_elements) - return None, header - - # API Server Performance Section - header.append(html.H3("🔌 API Server Performance")) - header.append(html.P("Kubernetes API server resource usage and request handling")) - - header += report.Plot_and_Text("Prom: Sutest ApiServer: CPU usage", args) - header += report.Plot_and_Text("Prom: Sutest ApiServer: Mem usage", args) - - header.append(html.Br()) - - # API Server Request Analysis Section - header.append(html.H3("📈 API Server Request Analysis")) - header.append(html.P("API request patterns, success rates, and error analysis")) - - header += report.Plot_and_Text("Prom: Sutest API Server Requests (successes)", args) - header += report.Plot_and_Text("Prom: Sutest API Server Requests (client errors)", args) - header += report.Plot_and_Text("Prom: Sutest API Server Requests (server errors)", args) - - header.append(html.Br()) - - if False: - # API Server Request Latency Section - header.append(html.H3("⏱️ API Server Request Latency")) - header.append(html.P("API request duration analysis by operation type")) - - header += report.Plot_and_Text("Prom: Sutest GET Requests duration", args) - header += report.Plot_and_Text("Prom: Sutest PUT Requests duration", args) - header += report.Plot_and_Text("Prom: Sutest LIST Requests duration", args) - header += report.Plot_and_Text("Prom: Sutest PATCH Requests duration", args) - - header.append(html.Br()) - - # ETCD Performance Section - header.append(html.H3("🗄️ ETCD Performance")) - header.append(html.P("ETCD cluster health and resource utilization")) - - header += report.Plot_and_Text("Prom: Sutest ETCD: CPU usage", args) - header += report.Plot_and_Text("Prom: Sutest ETCD: Mem usage", args) - - return None, header diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/report.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/report.py deleted file mode 100644 index 217b55dd0..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/report.py +++ /dev/null @@ -1,506 +0,0 @@ -from dash import html - -import projects.matrix_benchmarking.visualizations.helpers.plotting.report as report - -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common - - -def register(): - GuidellmResultsTable() - GuidellmPerformanceAnalysisReport() - - # Register Prometheus reports - from . import prometheus_reports - - prometheus_reports.register() - - -class GuidellmResultsTable: - def __init__(self): - self.name = "report: GuideLLM Results Table" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate GuideLLM results tables - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("GuideLLM Benchmark Results Tables")) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Add artifact links at the top - header.append(html.H4("🔗 Artifact Links")) - artifact_links = [] - - for i, entry in enumerate(entries): - # Get artifacts base directory for links - artifacts_basedir = getattr(entry.results, "from_local_env", {}) - if hasattr(artifacts_basedir, "artifacts_basedir"): - artifacts_basedir = artifacts_basedir.artifacts_basedir - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) if variables else f"Test {i + 1}" - artifact_links.append( - html.Li( - [ - html.A( - f"🗂️ {entry_name} - Test artifacts directory", - href=str(artifacts_basedir), - target="_blank", - ) - ] - ) - ) - - # Look for specific GuideLLM benchmark directory and log file - if ( - hasattr(entry.results, "guidellm_benchmarks") - and entry.results.guidellm_benchmarks - ): - guidellm_dir = None - log_file = None - - # Look for directory matching pattern: *__llmd__run_guidellm_benchmark - for item in artifacts_basedir.iterdir(): - if item.is_dir() and "__llmd__run_guidellm_benchmark" in item.name: - guidellm_dir = item - # Log file is directly in this directory - potential_log = ( - guidellm_dir / "artifacts" / "guidellm_benchmark_job.logs" - ) - if potential_log.exists(): - log_file = potential_log - break - - if guidellm_dir: - guidellm_sublinks = [ - html.Li( - html.A( - "📊 GuideLLM benchmark directory", - href=str(guidellm_dir), - target="_blank", - ) - ) - ] - if log_file: - guidellm_sublinks.append( - html.Li( - html.A("📄 benchmark logs", href=str(log_file), target="_blank") - ) - ) - artifact_links.append(html.Li(["GuideLLM:", html.Ul(guidellm_sublinks)])) - else: - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) if variables else f"Test {i + 1}" - artifact_links.append(html.Li(f"🗂️ {entry_name} - Artifacts not available")) - - if artifact_links: - header.append(html.Ul(artifact_links)) - else: - header.append(html.P("No artifact links available")) - - header.append(html.Br()) - - # Collect all GuideLLM benchmarks across all entries with their configuration info - all_benchmarks = [] - benchmark_configs = [] - for entry in entries: - if hasattr(entry.results, "guidellm_benchmarks") and entry.results.guidellm_benchmarks: - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) if variables else f"Unknown Configuration" - for benchmark in entry.results.guidellm_benchmarks: - all_benchmarks.append(benchmark) - benchmark_configs.append(entry_name) - - if not all_benchmarks: - header.append(html.P("No GuideLLM benchmark data found.")) - return None, header - - # Sort benchmarks by configuration first, then by request rate (best performance first) - # Create pairs of (benchmark, config) and sort them - benchmark_pairs = list(zip(all_benchmarks, benchmark_configs)) - benchmark_pairs.sort(key=lambda pair: (pair[1], -pair[0].request_rate)) - all_benchmarks = [pair[0] for pair in benchmark_pairs] - benchmark_configs = [pair[1] for pair in benchmark_pairs] - - # Create performance overview table - header.append(html.H3("📊 Performance Overview")) - - overview_headers = [ - "Configuration", - "Strategy", - "Request Rate (req/s)", - "TTFT P50 (ms)", - "TTFT P95 (ms)", - "Latency P50 (ms)", - "Latency P95 (ms)", - "Tokens/s", - "Concurrency", - ] - - overview_rows = [] - for i, benchmark in enumerate(all_benchmarks): - row = [ - benchmark_configs[i], - benchmark.strategy, - f"{benchmark.request_rate:.2f}", - f"{benchmark.ttft_median:.1f}", - f"{benchmark.ttft_p95:.1f}", - f"{benchmark.request_latency_median * 1000:.1f}", - f"{benchmark.request_latency_p95 * 1000:.1f}", - f"{benchmark.tokens_per_second:.1f}", - f"{benchmark.request_concurrency:.1f}", - ] - overview_rows.append(row) - - # Create HTML table - header.append(self._create_html_table(overview_headers, overview_rows)) - header.append(html.Br()) - - # Create latency breakdown table - header.append(html.H3("⏱️ Latency Component Breakdown")) - - latency_headers = [ - "Configuration", - "Strategy", - "TTFT P50", - "TTFT P95", - "ITL P50", - "ITL P95", - "TPOT P50", - "TPOT P95", - "Total Latency P50", - "Total Latency P95", - ] - - latency_rows = [] - for i, benchmark in enumerate(all_benchmarks): - row = [ - benchmark_configs[i], - benchmark.strategy, - f"{benchmark.ttft_median:.1f} ms", - f"{benchmark.ttft_p95:.1f} ms", - f"{benchmark.itl_median:.1f} ms", - f"{benchmark.itl_p95:.1f} ms", - f"{benchmark.tpot_median:.1f} ms", - f"{benchmark.tpot_p95:.1f} ms", - f"{benchmark.request_latency_median * 1000:.1f} ms", - f"{benchmark.request_latency_p95 * 1000:.1f} ms", - ] - latency_rows.append(row) - - header.append(self._create_html_table(latency_headers, latency_rows)) - header.append(html.Br()) - - # Create token throughput table - header.append(html.H3("🔤 Token Throughput Analysis")) - header.append(html.H3("🔤 Token Throughput Analysis")) - - token_headers = [ - "Configuration", - "Strategy", - "Total Tokens/s", - "Input Tokens/s", - "Output Tokens/s", - "Input Tokens/req", - "Output Tokens/req", - "Total Tokens/req", - ] - - token_rows = [] - for i, benchmark in enumerate(all_benchmarks): - row = [ - benchmark_configs[i], - benchmark.strategy, - f"{benchmark.tokens_per_second:.1f}", - f"{benchmark.input_tokens_per_second:.1f}", - f"{benchmark.output_tokens_per_second:.1f}", - f"{benchmark.input_tokens_per_request:.1f}", - f"{benchmark.output_tokens_per_request:.1f}", - f"{benchmark.total_tokens_per_request:.1f}", - ] - token_rows.append(row) - - header.append(self._create_html_table(token_headers, token_rows)) - header.append(html.Br()) - - # Add summary statistics - header.append(html.H3("📈 Summary Statistics")) - - # Find best performers with their configurations - best_throughput_idx = max( - range(len(all_benchmarks)), key=lambda i: all_benchmarks[i].request_rate - ) - best_latency_idx = min( - range(len(all_benchmarks)), key=lambda i: all_benchmarks[i].ttft_median - ) - best_tokens_idx = max( - range(len(all_benchmarks)), key=lambda i: all_benchmarks[i].tokens_per_second - ) - - best_throughput = all_benchmarks[best_throughput_idx] - best_latency = all_benchmarks[best_latency_idx] - best_tokens = all_benchmarks[best_tokens_idx] - - # Configuration analysis - configurations = {} - for i, benchmark in enumerate(all_benchmarks): - config = benchmark_configs[i] - if config not in configurations: - configurations[config] = {"throughput": [], "latency": [], "tokens": [], "count": 0} - configurations[config]["throughput"].append(benchmark.request_rate) - configurations[config]["latency"].append(benchmark.ttft_median) - configurations[config]["tokens"].append(benchmark.tokens_per_second) - configurations[config]["count"] += 1 - - summary_stats = [ - f"🏆 Best throughput: {best_throughput.strategy} in {benchmark_configs[best_throughput_idx]} ({best_throughput.request_rate:.2f} req/s)", - f"⚡ Best TTFT: {best_latency.strategy} in {benchmark_configs[best_latency_idx]} ({best_latency.ttft_median:.1f} ms)", - f"🚀 Best token throughput: {best_tokens.strategy} in {benchmark_configs[best_tokens_idx]} ({best_tokens.tokens_per_second:.1f} tok/s)", - f"📊 Total strategies tested: {len(all_benchmarks)} across {len(configurations)} configurations", - ] - - for stat in summary_stats: - header.append(html.P(stat)) - - # Add per-configuration summary - if len(configurations) > 1: - header.append(html.Br()) - header.append(html.H4("🔧 Configuration Performance Comparison")) - - for config, metrics in configurations.items(): - avg_throughput = sum(metrics["throughput"]) / len(metrics["throughput"]) - avg_latency = sum(metrics["latency"]) / len(metrics["latency"]) - avg_tokens = sum(metrics["tokens"]) / len(metrics["tokens"]) - - config_summary = f"• {config}: {avg_throughput:.1f} req/s avg, {avg_latency:.1f} ms TTFT avg, {avg_tokens:.0f} tok/s avg ({metrics['count']} strategies)" - header.append(html.P(config_summary)) - - return None, header - - def _create_html_table(self, headers, rows): - """Create an HTML table with the given headers and rows""" - table_style = { - "border": "1px solid #ddd", - "border-collapse": "collapse", - "width": "100%", - "margin": "10px 0", - } - - header_style = { - "background-color": "#f2f2f2", - "padding": "12px", - "text-align": "left", - "border": "1px solid #ddd", - "font-weight": "bold", - } - - cell_style = {"padding": "8px", "text-align": "left", "border": "1px solid #ddd"} - - # Create header row - header_cells = [html.Th(header, style=header_style) for header in headers] - header_row = html.Tr(header_cells) - - # Create data rows - data_rows = [] - for i, row in enumerate(rows): - # Alternate row colors for better readability - row_style = {"background-color": "#f9f9f9"} if i % 2 == 0 else {} - - cells = [html.Td(cell, style={**cell_style, **row_style}) for cell in row] - data_rows.append(html.Tr(cells)) - - return html.Table([header_row] + data_rows, style=table_style) - - -class GuidellmPerformanceAnalysisReport: - def __init__(self): - self.name = "report: GuideLLM Performance Analysis" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate focused report showing token throughput and TTFT performance analysis - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("📊 GuideLLM: Performance Analysis")) - header.append( - html.P( - "Comprehensive performance analysis including token throughput scaling and TTFT latency patterns across different test configurations" - ) - ) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Check if we have GuideLLM benchmark data - has_guidellm_data = False - for entry in entries: - if hasattr(entry.results, "guidellm_benchmarks") and entry.results.guidellm_benchmarks: - has_guidellm_data = True - break - - if not has_guidellm_data: - header.append(html.P("⚠️ No GuideLLM benchmark data available.")) - header.append( - html.P( - "This report requires GuideLLM benchmark results to analyze token throughput scaling." - ) - ) - header.append( - html.P( - "To enable benchmarks, set tests.llmd.benchmarks.guidellm.enabled: true in the configuration." - ) - ) - return None, header - - # Token Throughput Analysis Section - header.append(html.H3("🚀 Token Throughput vs Concurrency")) - header.append( - html.P("Analysis of how token generation throughput scales with concurrency levels") - ) - header += report.Plot_and_Text("Guidellm Tokens vs Concurrency", args) - header += report.Plot_and_Text("Token Throughput Percentiles Analysis", args) - - # TTFT Analysis Section - header.append(html.H3("⚡ Time to First Token (TTFT) Analysis")) - header.append( - html.P( - "Latency analysis showing response time percentiles across different concurrency levels" - ) - ) - header += report.Plot_and_Text("Guidellm TTFT Analysis", args) - - # TPOT Analysis Section - header.append(html.H3("🔄 Time Per Output Token (TPOT) Analysis")) - header.append( - html.P( - "Token generation speed analysis showing how quickly individual tokens are produced" - ) - ) - header += report.Plot_and_Text("Guidellm TPOT Analysis", args) - - # ITL Analysis Section - header.append(html.H3("⏱️ Inter-Token Latency (ITL) Analysis")) - header.append( - html.P("Streaming responsiveness analysis showing the delay between consecutive tokens") - ) - header += report.Plot_and_Text("Guidellm ITL Analysis", args) - - # E2E Latency Analysis Section - header.append(html.H3("🎯 End-to-End (E2E) Latency Analysis")) - header.append( - html.P( - "Complete request duration analysis from request initiation to final response completion" - ) - ) - header += report.Plot_and_Text("Guidellm E2E Latency Analysis", args) - - # Add summary analysis section - header.append(html.H3("📈 Key Insights")) - - # Collect all GuideLLM benchmarks for analysis - all_benchmarks = [] - benchmark_configs = [] - for entry in entries: - if hasattr(entry.results, "guidellm_benchmarks") and entry.results.guidellm_benchmarks: - entry_name = entry.get_name(variables) if variables else f"Unknown Configuration" - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy != "throughput": # Skip throughput-only strategies - all_benchmarks.append(benchmark) - benchmark_configs.append(entry_name) - - if all_benchmarks: - # Find best token throughput performers - best_tokens_idx = max( - range(len(all_benchmarks)), key=lambda i: all_benchmarks[i].tokens_per_second - ) - best_efficiency_idx = max( - range(len(all_benchmarks)), - key=lambda i: ( - all_benchmarks[i].tokens_per_second - / max(all_benchmarks[i].request_concurrency, 1) - ), - ) - - best_tokens = all_benchmarks[best_tokens_idx] - best_efficiency = all_benchmarks[best_efficiency_idx] - - # Configuration analysis - configurations = {} - for i, benchmark in enumerate(all_benchmarks): - config = benchmark_configs[i] - if config not in configurations: - configurations[config] = { - "max_tokens": 0, - "optimal_concurrency": 0, - "strategies": 0, - } - - if benchmark.tokens_per_second > configurations[config]["max_tokens"]: - configurations[config]["max_tokens"] = benchmark.tokens_per_second - configurations[config]["optimal_concurrency"] = benchmark.request_concurrency - - configurations[config]["strategies"] += 1 - - # Sort configurations by performance - sorted_configs = sorted( - configurations.items(), key=lambda x: x[1]["max_tokens"], reverse=True - ) - - insights = [ - f"🏆 Best token throughput: {best_tokens.strategy} in {benchmark_configs[best_tokens_idx]} ({best_tokens.tokens_per_second:.0f} tok/s)", - f"⚡ Most efficient: {best_efficiency.strategy} in {benchmark_configs[best_efficiency_idx]} ({(best_efficiency.tokens_per_second / max(best_efficiency.request_concurrency, 1)):.0f} tok/s per concurrency unit)", - f"📊 Total configurations tested: {len(configurations)}", - f"🎯 Total strategies analyzed: {len(all_benchmarks)}", - ] - - for insight in insights: - header.append(html.P(insight)) - - if len(sorted_configs) > 1: - header.append(html.Br()) - header.append(html.H4("🔧 Configuration Performance Ranking")) - - for i, (config, metrics) in enumerate(sorted_configs): - rank_emoji = ["🥇", "🥈", "🥉"][i] if i < 3 else "📍" - header.append( - html.P( - f"{rank_emoji} {config}: {metrics['max_tokens']:.0f} tok/s (optimal concurrency: {metrics['optimal_concurrency']:.0f}, {metrics['strategies']} strategies)" - ) - ) - - header.append(html.Br()) - header.append( - html.P( - "💡 Use this analysis to identify optimal concurrency settings for maximum token generation throughput in your specific deployment configuration." - ) - ) - - else: - header.append(html.P("No benchmark data available for analysis.")) - - return None, header diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_analysis.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_analysis.py deleted file mode 100644 index cb9d2af3a..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_analysis.py +++ /dev/null @@ -1,1381 +0,0 @@ -import pandas as pd -import numpy as np -import plotly.express as px -import plotly.graph_objects as go -from dash import html - -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common - - -def _get_plot_title_with_context_info(base_title, variables, settings): - """ - Helper function to add context info (model, load_shape) to plot titles if not in variables - - Args: - base_title: The base title/subtitle (e.g., "Token Throughput vs Concurrency
    Higher is better") - variables: Dict of variables being varied in the test - settings: Test settings object - - Returns: - Updated title with context info appended if needed - """ - context_info = [] - - # settings to show in the subtitle if not part of the 'variables' - SUBTITLE_SETTINGS = ( - "model", - "flavor", - "load_shape", - ) - - for setting_name in SUBTITLE_SETTINGS: - if setting_name in variables: - continue - - setting_value = settings.get(setting_name) - if not setting_value: - continue - context_info.append(f"{setting_name.replace('_', ' ').title()}: {setting_value}") - - if not context_info: - return base_title - - context_str = " • ".join(context_info) - # Insert context info before the closing tag or at the end - if "
    " in base_title and "" in base_title: - # Insert before closing - return base_title.replace("", f" • {context_str}") - else: - # Append as subtitle - return f"{base_title}
    {context_str}" - - -def register(): - GuidellmThroughputScaling() - GuidellmLatencyVsThroughput() - GuidellmLatencyOverview() - GuidellmTokensConcurrency() - GuidellmTTFTAnalysis() - GuidellmTPOTAnalysis() - GuidellmITLAnalysis() - GuidellmE2ELatencyAnalysis() - - TokenThroughputAnalysis() - TokenThroughputPercentilesAnalysis() - - -class GuidellmThroughputScaling: - def __init__(self): - self.name = "Guidellm Throughput Scaling" - self.id_name = "guidellm_throughput_scaling" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Plot request rate vs concurrency to show scaling behavior - """ - ## rewrite below - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - # Include all strategies - let's show the full picture - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - # Determine strategy type for better grouping - strategy_type = "Other" - if "constant" in benchmark.strategy.lower(): - if "arrival_rate" in benchmark.strategy.lower(): - strategy_type = "Constant Arrival Rate" - elif "concurrency" in benchmark.strategy.lower(): - strategy_type = "Constant Concurrency" - else: - strategy_type = "Constant" - elif "sweep" in benchmark.strategy.lower(): - strategy_type = "Sweep" - elif "ramp" in benchmark.strategy.lower(): - strategy_type = "Ramp" - - data.append( - { - "Test Configuration": entry_name, - "Concurrency": benchmark.request_concurrency, - "Request Rate (req/s)": benchmark.request_rate, - "Strategy": benchmark.strategy, - "Strategy Type": strategy_type, - "TTFT (ms)": benchmark.ttft_median, - "Latency (ms)": benchmark.request_latency_median * 1000, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # Sort by Concurrency for proper plot ordering - df = df.sort_values("Concurrency") - - # 2. Generate plotly express plot - title = _get_plot_title_with_context_info( - "Request Throughput vs Concurrency by Configuration", - variables, - settings, - ) - - fig = px.scatter( - df, - hover_data=df.columns, - x="Concurrency", - y="Request Rate (req/s)", - color="Test Configuration", - symbol="Strategy Type", - size="Tokens/s", - text="Strategy", - title=title, - ) - - fig.update_traces(textposition="top center") - fig.update_layout(showlegend=True) - fig.update_yaxes(rangemode="tozero") # Ensure 0 is always shown on y-axis - - # Add trend lines for constant strategies - constant_data = df[df["Strategy Type"].str.contains("Constant", na=False)] - if len(constant_data) > 1: - fig.add_scatter( - x=constant_data["Concurrency"], - y=constant_data["Request Rate (req/s)"], - mode="lines", - name="Constant Strategy Trend", - line=dict(dash="dash", color="gray"), - showlegend=True, - ) - - # 3. Generate summary text - max_rate = df["Request Rate (req/s)"].max() - min_rate = df["Request Rate (req/s)"].min() - avg_rate = df["Request Rate (req/s)"].mean() - - best_strategy = df.loc[df["Request Rate (req/s)"].idxmax()] - - # Configuration breakdown - config_counts = df["Test Configuration"].value_counts() - strategy_counts = df["Strategy Type"].value_counts() - total_strategies = len(df) - - performance_ratio = ((max_rate - min_rate) / min_rate * 100) if min_rate > 0 else 0 - - msg = [] - msg.append( - f"Showing {total_strategies} strategies across {len(config_counts)} test configurations:" - ) - msg.append(html.Br()) - for config, count in config_counts.items(): - msg.append(f"• {config}: {count} strategies") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append(f"Strategy types: {', '.join(strategy_counts.keys())}") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - f"Best performing: {best_strategy['Strategy']} in {best_strategy['Test Configuration']} ({max_rate:.2f} req/s)" - ) - msg.append(html.Br()) - msg.append( - f"Performance range: {min_rate:.2f} - {max_rate:.2f} req/s ({performance_ratio:.1f}% spread)" - ) - msg.append(html.Br()) - msg.append( - "Note: Bubble size shows token throughput, color shows test configuration, shape shows strategy type" - ) - - # 4. Return fig, msg - return fig, msg - - -class GuidellmLatencyVsThroughput: - def __init__(self): - self.name = "Guidellm Latency vs Throughput" - self.id_name = "guidellm_latency_vs_throughput" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Plot latency vs throughput trade-off - """ - ## rewrite below - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - data.append( - { - "Test Configuration": entry_name, - "Request Rate (req/s)": benchmark.request_rate, - "Latency (ms)": benchmark.request_latency_median * 1000, # Convert to ms - "Strategy": benchmark.strategy, - "TTFT (ms)": benchmark.ttft_median, - "Concurrency": benchmark.request_concurrency, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # Sort by Concurrency for consistent ordering - df = df.sort_values("Concurrency") - - # 2. Generate plotly express plot - title = _get_plot_title_with_context_info( - "Latency vs Throughput Trade-off by Configuration", - variables, - settings, - ) - - fig = px.scatter( - df, - hover_data=df.columns, - x="Request Rate (req/s)", - y="Latency (ms)", - color="Test Configuration", - symbol="Test Configuration", - size="Tokens/s", - text="Strategy", - title=title, - ) - - fig.update_traces(textposition="top center") - fig.update_layout(showlegend=True) - - # 3. Generate summary text - best_throughput = df.loc[df["Request Rate (req/s)"].idxmax()] - best_latency = df.loc[df["Latency (ms)"].idxmin()] - - # Calculate efficiency (high throughput, low latency) - df["Efficiency"] = df["Request Rate (req/s)"] / df["Latency (ms)"] - most_efficient = df.loc[df["Efficiency"].idxmax()] - - msg = [] - msg.append( - f"Highest throughput: {best_throughput['Strategy']} in {best_throughput['Test Configuration']} ({best_throughput['Request Rate (req/s)']:.2f} req/s)" - ) - msg.append(html.Br()) - msg.append( - f"Lowest latency: {best_latency['Strategy']} in {best_latency['Test Configuration']} ({best_latency['Latency (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append( - f"Most efficient: {most_efficient['Strategy']} in {most_efficient['Test Configuration']} ({most_efficient['Efficiency']:.2f} req/s per ms)" - ) - msg.append(html.Br()) - msg.append("Note: Higher throughput and lower latency indicate better performance") - - # 4. Return fig, msg - return fig, msg - - -class GuidellmLatencyOverview: - def __init__(self): - self.name = "Guidellm Latency Overview" - self.id_name = "guidellm_latency_overview" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Performance overview focusing on latency metrics across strategies - """ - ## rewrite below - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - for benchmark in entry.results.guidellm_benchmarks: - data.append( - { - "Test Configuration": entry_name, - "Strategy": benchmark.strategy, - "Full Strategy Name": f"{benchmark.strategy} ({entry_name})", - "Request Latency (ms)": benchmark.request_latency_median * 1000, - "TTFT (ms)": benchmark.ttft_median, - "Concurrency": benchmark.request_concurrency, - "Request Rate": benchmark.request_rate, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # Sort by request latency for better visualization - df = df.sort_values("Request Latency (ms)") - - # 2. Generate plotly express plot - title = _get_plot_title_with_context_info( - "Latency Overview by Strategy and Configuration", - variables, - settings, - ) - - fig = px.bar( - df, - hover_data=df.columns, - x="Full Strategy Name", - y="Request Latency (ms)", - color="Test Configuration", - text="Request Latency (ms)", - title=title, - ) - - fig.update_traces(texttemplate="%{text:.1f}ms", textposition="outside") - fig.update_layout(showlegend=True, xaxis_tickangle=-45) - - # 3. Generate summary text - best_latency = df.loc[df["Request Latency (ms)"].idxmin()] - worst_latency = df.loc[df["Request Latency (ms)"].idxmax()] - avg_latency = df["Request Latency (ms)"].mean() - - best_ttft = df.loc[df["TTFT (ms)"].idxmin()] - avg_ttft = df["TTFT (ms)"].mean() - - latency_spread = ( - worst_latency["Request Latency (ms)"] - best_latency["Request Latency (ms)"] - ) - - # Configuration performance summary - config_latency = ( - df.groupby("Test Configuration")["Request Latency (ms)"].mean().sort_values() - ) - best_config = config_latency.index[0] - worst_config = config_latency.index[-1] - - msg = [] - msg.append( - f"Best request latency: {best_latency['Strategy']} in {best_latency['Test Configuration']} ({best_latency['Request Latency (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append( - f"Worst request latency: {worst_latency['Strategy']} in {worst_latency['Test Configuration']} ({worst_latency['Request Latency (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(f"Average request latency: {avg_latency:.1f} ms") - msg.append(html.Br()) - msg.append(f"Latency spread: {latency_spread:.1f} ms") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - f"Best configuration overall: {best_config} ({config_latency[best_config]:.1f} ms avg)" - ) - msg.append(html.Br()) - msg.append( - f"Best TTFT: {best_ttft['Strategy']} in {best_ttft['Test Configuration']} ({best_ttft['TTFT (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(f"Average TTFT: {avg_ttft:.1f} ms") - msg.append(html.Br()) - msg.append("Note: Bar color shows test configuration") - - # 4. Return fig, msg - return fig, msg - - -class GuidellmTokensConcurrency: - def __init__(self): - self.name = "Guidellm Tokens vs Concurrency" - self.id_name = "guidellm_tokens_concurrency" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Plot tokens per second vs concurrency to show token throughput scaling behavior - """ - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 2. Generate plotly express plot with consistent color scheme - use_platform_markers = cfg.get("markers_by", None) == "platform" - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - # Extract platform information - platform = getattr(entry.settings, "platform", "unknown") - - # Include all strategies - let's show the full picture - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - # Determine strategy type for better grouping - strategy_type = "Other" - if "constant" in benchmark.strategy.lower(): - if "arrival_rate" in benchmark.strategy.lower(): - strategy_type = "Constant Arrival Rate" - elif "concurrency" in benchmark.strategy.lower(): - strategy_type = "Constant Concurrency" - else: - strategy_type = "Constant" - elif "sweep" in benchmark.strategy.lower(): - strategy_type = "Sweep" - elif "ramp" in benchmark.strategy.lower(): - strategy_type = "Ramp" - - # Use mean values for throughput analysis - total_tokens_mean = getattr( - benchmark, - "tokens_per_second_mean", - getattr(benchmark, "tokens_per_second", benchmark.tokens_per_second), - ) - output_tokens_mean = getattr( - benchmark, - "output_tokens_per_second_mean", - getattr( - benchmark, "output_tokens_per_second", benchmark.output_tokens_per_second - ), - ) - input_tokens_mean = getattr( - benchmark, - "input_tokens_per_second_mean", - getattr( - benchmark, "input_tokens_per_second", benchmark.input_tokens_per_second - ), - ) - - data.append( - { - "Test Configuration": entry_name, - "Platform": platform, - "Concurrency": benchmark.request_concurrency, - "Tokens/s": output_tokens_mean, - "Input Tokens/s": input_tokens_mean, - "Output Tokens/s": output_tokens_mean, - "Total Tokens/s": total_tokens_mean, - "Strategy": benchmark.strategy, - "Strategy Type": strategy_type, - "Request Rate (req/s)": benchmark.request_rate, - "TTFT (ms)": benchmark.ttft_median, - "Latency (ms)": benchmark.request_latency_median * 1000, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # Sort by Concurrency for proper plot ordering - df = df.sort_values(["Concurrency", "Test Configuration"]) - - # Create title with load_shape if missing from variables - title = _get_plot_title_with_context_info( - "Token Throughput vs Concurrency by Configuration
    Higher is better • Mean values", - variables, - settings, - ) - - # Set line dash parameter conditionally for platforms - line_dash = "Platform" if use_platform_markers else None - - fig = px.line( - df, - hover_data=df.columns, - x="Concurrency", - y="Tokens/s", - color="Test Configuration", - line_dash=line_dash, - markers=True, - title=title, - ) - - fig.update_traces(mode="lines+markers") - fig.update_layout(showlegend=True) - fig.update_yaxes(rangemode="tozero") # Ensure 0 is always shown on y-axis - - # 3. Generate summary text - max_tokens = df["Tokens/s"].max() - min_tokens = df["Tokens/s"].min() - avg_tokens = df["Tokens/s"].mean() - - best_tokens = df.loc[df["Tokens/s"].idxmax()] - best_efficiency = df.loc[(df["Tokens/s"] / df["Concurrency"]).idxmax()] - - # Configuration breakdown - config_counts = df["Test Configuration"].value_counts() - strategy_counts = df["Strategy Type"].value_counts() - total_strategies = len(df) - - # Performance analysis - token_improvement = ((max_tokens - min_tokens) / min_tokens * 100) if min_tokens > 0 else 0 - - # Find optimal concurrency ranges - config_optimal = {} - for config in df["Test Configuration"].unique(): - config_df = df[df["Test Configuration"] == config] - if len(config_df) > 0: - best_idx = config_df["Tokens/s"].idxmax() - optimal_concurrency = config_df.loc[best_idx, "Concurrency"] - optimal_tokens = config_df.loc[best_idx, "Tokens/s"] - config_optimal[config] = (optimal_concurrency, optimal_tokens) - - msg = [] - msg.append( - f"Analyzing {total_strategies} strategies across {len(config_counts)} test configurations:" - ) - msg.append(html.Br()) - for config, count in config_counts.items(): - msg.append(f"• {config}: {count} strategies") - msg.append(html.Br()) - msg.append(html.Br()) - - msg.append( - f"Best token throughput: {best_tokens['Strategy']} in {best_tokens['Test Configuration']} ({max_tokens:.0f} tok/s at concurrency {best_tokens['Concurrency']:.0f})" - ) - msg.append(html.Br()) - msg.append( - f"Token throughput range: {min_tokens:.0f} - {max_tokens:.0f} tok/s ({token_improvement:.1f}% improvement)" - ) - msg.append(html.Br()) - msg.append( - f"Most efficient: {best_efficiency['Strategy']} in {best_efficiency['Test Configuration']} ({(best_efficiency['Tokens/s'] / best_efficiency['Concurrency']):.0f} tok/s per concurrency unit)" - ) - msg.append(html.Br()) - msg.append(html.Br()) - - if config_optimal: - msg.append("Optimal concurrency points by configuration:") - msg.append(html.Br()) - for config, (opt_conc, opt_tokens) in config_optimal.items(): - msg.append(f"• {config}: {opt_conc:.0f} concurrency → {opt_tokens:.0f} tok/s") - msg.append(html.Br()) - msg.append(html.Br()) - - if use_platform_markers: - msg.append( - "Note: Color shows test configuration, line style shows platform, lines connect points for each configuration. Values shown are mean for average performance analysis." - ) - else: - msg.append( - "Note: Color shows test configuration, lines connect points for each configuration. Values shown are mean for average performance analysis." - ) - - # 4. Return fig, msg - return fig, msg - - -# Base class for latency analysis to eliminate code duplication -class GuidellmLatencyAnalysisBase: - def __init__(self, metric_config): - self.metric_config = metric_config - self.name = f"Guidellm {metric_config['name']} Analysis" - self.id_name = f"guidellm_{metric_config['name'].lower()}_analysis" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Generic plot for latency percentiles analysis across different strategies and configurations - """ - config = self.metric_config - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - # Get values with unit conversion - p50_value = getattr(benchmark, config["p50_field"]) * config["unit_conversion"] - p95_value = getattr(benchmark, config["p95_field"]) * config["unit_conversion"] - - data.append( - { - "Test Configuration": entry_name, - "Strategy": benchmark.strategy, - "Full Strategy Name": f"{benchmark.strategy} ({entry_name})", - "Concurrency": benchmark.request_concurrency, - "Request Rate (req/s)": benchmark.request_rate, - f"{config['name']} P50 (ms)": p50_value, - f"{config['name']} P95 (ms)": p95_value, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # 2. Generate plotly subplots for P50 and P95 - from plotly.subplots import make_subplots - - fig = make_subplots( - rows=1, - cols=2, - subplot_titles=(f"{config['name']} P50 (Median)", f"{config['name']} P95"), - shared_xaxes=True, - horizontal_spacing=0.1, - ) - - # Get unique configurations and colors (consistent with other plots) - configurations = sorted(df["Test Configuration"].unique()) - available_colors = px.colors.qualitative.Set1 - color_map = { - config_name: available_colors[i % len(available_colors)] - for i, config_name in enumerate(configurations) - } - - for i, config_name in enumerate(configurations): - config_df = df[df["Test Configuration"] == config_name].sort_values("Concurrency") - - # Add P50 subplot (left) - fig.add_trace( - go.Scatter( - x=config_df["Concurrency"], - y=config_df[f"{config['name']} P50 (ms)"], - mode="lines+markers", - name=f"{config_name}", - line=dict(color=color_map[config_name], width=2), - hovertemplate=f"{config_name}
    " - + "Concurrency: %{x}
    " - + f"{config['name']} P50: %{{y:.1f}} ms
    " - + "", - showlegend=True, - ), - row=1, - col=1, - ) - - # Add P95 subplot (right) - fig.add_trace( - go.Scatter( - x=config_df["Concurrency"], - y=config_df[f"{config['name']} P95 (ms)"], - mode="lines+markers", - name=f"{config_name}", - line=dict(color=color_map[config_name], width=2), - hovertemplate=f"{config_name}
    " - + "Concurrency: %{x}
    " - + f"{config['name']} P95: %{{y:.1f}} ms
    " - + "", - showlegend=False, # Don't duplicate legend - ), - row=1, - col=2, - ) - - # Create title with load_shape if missing from variables - title = _get_plot_title_with_context_info( - f"{config['description']} Analysis by Concurrency
    Lower is better", - variables, - settings, - ) - - fig.update_layout(title_text=title, showlegend=True, hovermode="closest", height=500) - - # Update subplot axis labels - fig.update_xaxes(title_text="Concurrency", row=1, col=1) - fig.update_xaxes(title_text="Concurrency", row=1, col=2) - fig.update_yaxes(title_text=f"{config['name']} P50 (ms)", row=1, col=1) - fig.update_yaxes(title_text=f"{config['name']} P95 (ms)", row=1, col=2) - - # 3. Generate summary text - p50_col = f"{config['name']} P50 (ms)" - p95_col = f"{config['name']} P95 (ms)" - - best_p50_idx = df[p50_col].idxmin() - best_p95_idx = df[p95_col].idxmin() - - best_p50 = df.loc[best_p50_idx] - best_p95 = df.loc[best_p95_idx] - - # Calculate percentile spreads - avg_p50 = df[p50_col].mean() - avg_p95 = df[p95_col].mean() - p50_range = df[p50_col].max() - df[p50_col].min() - p95_range = df[p95_col].max() - df[p95_col].min() - - # Configuration analysis - config_performance = ( - df.groupby("Test Configuration") - .agg({p50_col: "mean", p95_col: "mean"}) - .sort_values(p50_col) - ) - - best_config_name = config_performance.index[0] - - msg = [] - msg.append( - f"🏆 Best {config['name']} P50: {best_p50['Strategy']} in {best_p50['Test Configuration']} ({best_p50[p50_col]:.1f} ms)" - ) - msg.append(html.Br()) - msg.append( - f"🏆 Best {config['name']} P95: {best_p95['Strategy']} in {best_p95['Test Configuration']} ({best_p95[p95_col]:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(f"📊 Average {config['name']} P50: {avg_p50:.1f} ms, P95: {avg_p95:.1f} ms") - msg.append(html.Br()) - msg.append( - f"📈 {config['name']} variability: P50 range {p50_range:.1f} ms, P95 range {p95_range:.1f} ms" - ) - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - f"🥇 Best configuration overall: {best_config_name} (P50: {config_performance.loc[best_config_name, p50_col]:.1f} ms, P95: {config_performance.loc[best_config_name, p95_col]:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - f"📝 Note: Left subplot shows P50 (median), right subplot shows P95. {config['note_text']}" - ) - msg.append(html.Br()) - msg.append(f"💡 {config['insight_text']}") - - # 4. Return fig, msg - return fig, msg - - -# Simplified TTFT analysis class showing only P50 -class GuidellmTTFTAnalysis: - def __init__(self): - self.name = "Guidellm TTFT Analysis" - self.id_name = "guidellm_ttft_analysis" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Generate TTFT P50 analysis plot - """ - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - entry_name = entry.get_name(variables) - platform = getattr(entry.settings, "platform", "unknown") - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - ttft_p50 = getattr(benchmark, "ttft_median", None) - - data.append( - { - "Test Configuration": entry_name, - "Platform": platform, - "Strategy": benchmark.strategy, - "Concurrency": benchmark.request_concurrency, - "TTFT P50 (ms)": ttft_p50, - "Request Rate (req/s)": benchmark.request_rate, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - df = df.sort_values(["Concurrency", "Test Configuration"]) - - # 2. Generate plot - use_platform_markers = cfg.get("markers_by", None) == "platform" - line_dash = "Platform" if use_platform_markers else None - - title = _get_plot_title_with_context_info( - "TTFT (P50) vs Concurrency
    Lower is better • Median values", - variables, - settings, - ) - - fig = px.line( - df, - hover_data=df.columns, - x="Concurrency", - y="TTFT P50 (ms)", - color="Test Configuration", - line_dash=line_dash, - markers=True, - title=title, - ) - - fig.update_layout(showlegend=True) - fig.update_yaxes(rangemode="tozero") - - # 3. Generate summary text - if "TTFT P50 (ms)" in df.columns and not df["TTFT P50 (ms)"].isna().all(): - best_ttft = df.loc[df["TTFT P50 (ms)"].idxmin()] - avg_ttft = df["TTFT P50 (ms)"].mean() - ttft_range = df["TTFT P50 (ms)"].max() - df["TTFT P50 (ms)"].min() - - msg = [] - msg.append( - f"🏆 Best TTFT P50: {best_ttft['Strategy']} in {best_ttft['Test Configuration']} ({best_ttft['TTFT P50 (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(f"📊 Average TTFT P50: {avg_ttft:.1f} ms") - msg.append(html.Br()) - msg.append(f"📈 TTFT range: {ttft_range:.1f} ms") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - "💡 Lower TTFT values indicate better responsiveness. Look for configurations with consistently low and stable TTFT across concurrency levels." - ) - else: - msg = ["No TTFT data available for analysis"] - - return fig, msg - - -class GuidellmTPOTAnalysis(GuidellmLatencyAnalysisBase): - def __init__(self): - super().__init__( - { - "name": "TPOT", - "description": "TPOT Latency", - "p50_field": "tpot_median", - "p95_field": "tpot_p95", - "unit_conversion": 1000, # Convert from seconds to ms - "note_text": "TPOT measures generation speed per token.", - "insight_text": "Lower TPOT values indicate faster token generation. Look for configurations with consistently low TPOT across concurrency levels.", - } - ) - - -# Simplified ITL analysis class showing only P50 -class GuidellmITLAnalysis: - def __init__(self): - self.name = "Guidellm ITL Analysis" - self.id_name = "guidellm_itl_analysis" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Generate ITL P50 analysis plot - """ - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - entry_name = entry.get_name(variables) - platform = getattr(entry.settings, "platform", "unknown") - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - itl_p50 = getattr(benchmark, "itl_median", None) - if itl_p50 is not None: - itl_p50 = itl_p50 * 1000 # Convert from seconds to ms - - data.append( - { - "Test Configuration": entry_name, - "Platform": platform, - "Strategy": benchmark.strategy, - "Concurrency": benchmark.request_concurrency, - "ITL P50 (ms)": itl_p50, - "Request Rate (req/s)": benchmark.request_rate, - "Tokens/s": benchmark.tokens_per_second, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - df = df.sort_values(["Concurrency", "Test Configuration"]) - - # 2. Generate plot - use_platform_markers = cfg.get("markers_by", None) == "platform" - line_dash = "Platform" if use_platform_markers else None - - title = _get_plot_title_with_context_info( - "ITL (P50) vs Concurrency
    Lower is better • Median values", - variables, - settings, - ) - - fig = px.line( - df, - hover_data=df.columns, - x="Concurrency", - y="ITL P50 (ms)", - color="Test Configuration", - line_dash=line_dash, - markers=True, - title=title, - ) - - fig.update_layout(showlegend=True) - fig.update_yaxes(rangemode="tozero") - - # 3. Generate summary text - if "ITL P50 (ms)" in df.columns and not df["ITL P50 (ms)"].isna().all(): - best_itl = df.loc[df["ITL P50 (ms)"].idxmin()] - avg_itl = df["ITL P50 (ms)"].mean() - itl_range = df["ITL P50 (ms)"].max() - df["ITL P50 (ms)"].min() - - msg = [] - msg.append( - f"🏆 Best ITL P50: {best_itl['Strategy']} in {best_itl['Test Configuration']} ({best_itl['ITL P50 (ms)']:.1f} ms)" - ) - msg.append(html.Br()) - msg.append(f"📊 Average ITL P50: {avg_itl:.1f} ms") - msg.append(html.Br()) - msg.append(f"📈 ITL range: {itl_range:.1f} ms") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - "💡 Lower ITL values indicate smoother token streaming. Look for configurations with consistently low ITL for better user experience." - ) - else: - msg = ["No ITL data available for analysis"] - - return fig, msg - - -class GuidellmE2ELatencyAnalysis(GuidellmLatencyAnalysisBase): - def __init__(self): - super().__init__( - { - "name": "E2E Latency", - "description": "End-to-End Request Latency", - "p50_field": "request_latency_median", - "p95_field": "request_latency_p95", - "unit_conversion": 1000, # Convert from seconds to ms - "note_text": "E2E latency measures complete request duration.", - "insight_text": "Lower E2E values indicate faster overall response times. Look for configurations with consistently low E2E latency for better user experience.", - } - ) - - -class TokenThroughputAnalysis: - def __init__(self): - self.name = "Token Throughput Analysis" - self.id_name = "token_throughput_analysis" - self.no_graph = False - self.is_report = False - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Analyze token generation throughput across strategies - """ - ## rewrite below - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - for benchmark in entry.results.guidellm_benchmarks: - # Use P50 (median) values for more representative throughput - total_tokens_p50 = getattr( - benchmark, "tokens_per_second_median", benchmark.tokens_per_second - ) - output_tokens_p50 = getattr( - benchmark, "output_tokens_per_second_median", benchmark.output_tokens_per_second - ) - input_tokens_p50 = getattr( - benchmark, "input_tokens_per_second_median", benchmark.input_tokens_per_second - ) - - data.append( - { - "Test Configuration": entry_name, - "Strategy": benchmark.strategy, - "Full Strategy Name": f"{benchmark.strategy} ({entry_name})", - "Input Tokens/s": input_tokens_p50, - "Output Tokens/s": output_tokens_p50, - "Total Tokens/s": total_tokens_p50, - "Request Rate": benchmark.request_rate, - "TTFT (ms)": benchmark.ttft_median, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - # Sort by total token throughput - df = df.sort_values("Total Tokens/s", ascending=False) - - # 2. Generate plotly express plot - title = _get_plot_title_with_context_info( - "Token Throughput by Strategy and Configuration
    P50 (median) values", - variables, - settings, - ) - - fig = px.bar( - df, - hover_data=df.columns, - x="Full Strategy Name", - y="Total Tokens/s", - color="Test Configuration", - text="Total Tokens/s", - title=title, - ) - - fig.update_traces(texttemplate="%{text:.0f}", textposition="outside") - fig.update_layout(showlegend=True, xaxis_tickangle=-45) - - # Add stacked breakdown as a secondary chart - # Create stacked data for input/output breakdown - df_stacked = df.melt( - id_vars=["Strategy"], - value_vars=["Input Tokens/s", "Output Tokens/s"], - var_name="Token Type", - value_name="Tokens/s", - ) - - # Create title with load_shape if missing from variables - breakdown_title = _get_plot_title_with_context_info( - "Token Throughput Breakdown by Strategy", - variables, - settings, - ) - - fig_stacked = px.bar( - df_stacked, x="Strategy", y="Tokens/s", color="Token Type", title=breakdown_title - ) - - fig_stacked.update_layout(xaxis_tickangle=-45) - - # Use the main chart for now - # 3. Generate summary text - max_tokens = df["Total Tokens/s"].max() - min_tokens = df["Total Tokens/s"].min() - avg_tokens = df["Total Tokens/s"].mean() - - best_strategy = df.loc[df["Total Tokens/s"].idxmax()] - improvement = ((max_tokens - min_tokens) / min_tokens * 100) if min_tokens > 0 else 0 - - # Configuration performance summary - config_tokens = ( - df.groupby("Test Configuration")["Total Tokens/s"].mean().sort_values(ascending=False) - ) - best_config = config_tokens.index[0] - - msg = [] - msg.append( - f"Highest token throughput: {best_strategy['Strategy']} in {best_strategy['Test Configuration']} ({max_tokens:.0f} tok/s)" - ) - msg.append(html.Br()) - msg.append(f"Lowest token throughput: {min_tokens:.0f} tok/s") - msg.append(html.Br()) - msg.append(f"Average token throughput: {avg_tokens:.0f} tok/s") - msg.append(html.Br()) - msg.append(f"Performance improvement: {improvement:.1f}% (best vs worst)") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - f"Best configuration overall: {best_config} ({config_tokens[best_config]:.0f} tok/s avg)" - ) - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - "Note: All token throughput values shown are mean values for average performance analysis." - ) - - # 4. Return fig, msg - return fig, msg - - -class TokenThroughputPercentilesAnalysis: - def __init__(self): - self.name = "Token Throughput Percentiles Analysis" - self.id_name = "token_throughput_percentiles_analysis" - self.no_graph = False - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): - """ - Plot output token throughput percentiles: P25, P50, P70, P90 - """ - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # 1. Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - # Get unique name for this entry (includes flavor info) - entry_name = entry.get_name(variables) - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - data.append( - { - "Test Configuration": entry_name, - "Concurrency": benchmark.request_concurrency, - "Strategy": benchmark.strategy, - # Output token percentiles and mean - "Output Tokens/s P10": benchmark.output_tokens_per_second_p10, - "Output Tokens/s P25": benchmark.output_tokens_per_second_p25, - "Output Tokens/s P50": benchmark.output_tokens_per_second_p50, - "Output Tokens/s P75": benchmark.output_tokens_per_second_p75, - "Output Tokens/s P90": benchmark.output_tokens_per_second_p90, - "Output Tokens/s Mean": getattr( - benchmark, - "output_tokens_per_second_mean", - benchmark.output_tokens_per_second, - ), - "Request Rate (req/s)": benchmark.request_rate, - } - ) - - if not data: - return None, ["No Guidellm benchmark data available"] - - df = pd.DataFrame(data) - - # Sort by Concurrency for proper plot ordering - - # 2. Generate plot for output token percentiles - fig = go.Figure() - - # Get unique configurations and colors (consistent with other plots) - configurations = sorted(df["Test Configuration"].unique()) - available_colors = px.colors.qualitative.Set1 - color_map = { - config_name: available_colors[i % len(available_colors)] - for i, config_name in enumerate(configurations) - } - - # Define percentiles and mean to plot with distinct line styles - percentiles_config = [ - {"percentile": "P10", "line_style": {"width": 2, "dash": "longdash"}, "opacity": 0.6}, - {"percentile": "P25", "line_style": {"width": 2, "dash": "dot"}, "opacity": 0.7}, - {"percentile": "P50", "line_style": {"width": 4, "dash": "solid"}, "opacity": 1.0}, - {"percentile": "P75", "line_style": {"width": 3, "dash": "dash"}, "opacity": 0.9}, - {"percentile": "P90", "line_style": {"width": 2, "dash": "dashdot"}, "opacity": 0.8}, - {"percentile": "Mean", "line_style": {"width": 4, "dash": "dot"}, "opacity": 1.0}, - ] - - # Add traces for each configuration and percentile - for config_name in configurations: - config_df = df[df["Test Configuration"] == config_name].sort_values("Concurrency") - - if len(config_df) == 0: - continue - - for perc_config in percentiles_config: - percentile = perc_config["percentile"] - column_name = f"Output Tokens/s {percentile}" - - if column_name not in config_df.columns: - continue - - config_df = config_df.sort_values(["Concurrency", column_name]) - - # Create unique trace name - trace_name = f"{config_name} - {percentile}" - - # Use base color - base_color = color_map[config_name] - - fig.add_trace( - go.Scatter( - x=config_df["Concurrency"], - y=config_df[column_name], - mode="lines+markers", - name=trace_name, - legendgroup=f"{config_name}", - marker=dict(size=6), - line=dict(color=base_color, **perc_config["line_style"]), - opacity=perc_config["opacity"], - hovertemplate=f"{config_name}
    " - + f"Output Tokens/s {percentile}
    " - + "Concurrency: %{x}
    " - + "Value: %{y:.0f} tok/s
    " - + "", - showlegend=True, - ) - ) - - # Create title with context info - title = _get_plot_title_with_context_info( - "Output Token Throughput Percentiles vs Concurrency
    Higher is better • P10 (long dash), P25 (dotted), P50 (solid), P75 (dashed), P90 (dash-dot), Mean (thick dotted)", - variables, - settings, - ) - - fig.update_layout( - title_text=title, - showlegend=True, - hovermode="closest", - height=700, - xaxis_title="Concurrency", - yaxis_title="Output Tokens per Second", - legend=dict( - groupclick="toggleitem", - title="Configuration - Percentile", - orientation="v", - x=1.02, - y=1, - font=dict(size=10), - ), - ) - fig.update_yaxes(rangemode="tozero") # Ensure 0 is always shown on y-axis - - # 3. Generate summary text focusing on output token percentiles - msg = [] - msg.append("🚀 Output Token Throughput Percentiles & Mean Analysis:") - msg.append(html.Br()) - - # Best performers for each percentile and mean - percentiles_to_analyze = ["P10", "P25", "P50", "P75", "P90", "Mean"] - best_performers = {} - - for perc in percentiles_to_analyze: - col_name = f"Output Tokens/s {perc}" - if col_name in df.columns: - best_idx = df[col_name].idxmax() - best_performers[perc] = { - "value": df.loc[best_idx, col_name], - "config": df.loc[best_idx, "Test Configuration"], - "strategy": df.loc[best_idx, "Strategy"], - "concurrency": df.loc[best_idx, "Concurrency"], - } - - # Show best performers - for perc in percentiles_to_analyze: - if perc in best_performers: - best = best_performers[perc] - msg.append( - f"• Best {perc}: {best['strategy']} in {best['config']} ({best['value']:.0f} tok/s at {best['concurrency']:.0f} concurrency)" - ) - msg.append(html.Br()) - - msg.append(html.Br()) - - # Performance consistency analysis - msg.append("📊 Performance Consistency:") - msg.append(html.Br()) - - # Calculate spreads between percentiles - for config_name in configurations: - config_df = df[df["Test Configuration"] == config_name] - if len(config_df) > 0: - avg_p90 = config_df["Output Tokens/s P90"].mean() - avg_p10 = config_df["Output Tokens/s P10"].mean() - spread = ((avg_p90 - avg_p10) / avg_p10 * 100) if avg_p10 > 0 else 0 - msg.append(f"• {config_name}: {spread:.1f}% spread (P90-P10)") - msg.append(html.Br()) - - msg.append(html.Br()) - msg.append("📈 Percentile Guide:") - msg.append(html.Br()) - msg.append("• P10: 10% of requests achieve this throughput or higher") - msg.append(html.Br()) - msg.append("• P25: 25% of requests achieve this throughput or higher") - msg.append(html.Br()) - msg.append("• P50: Median performance — 50% achieve this or higher") - msg.append(html.Br()) - msg.append("• P75: Good performance — 75% achieve this or higher") - msg.append(html.Br()) - msg.append("• P90: High performance — 90% achieve this or higher") - msg.append(html.Br()) - msg.append("• Mean: Average performance across all requests") - msg.append(html.Br()) - msg.append(html.Br()) - msg.append( - "💡 Analysis: Smaller spreads between percentiles indicate more consistent performance. Compare mean vs P50 to understand distribution skew. Focus on configurations with high P50 and small P90—P10 gaps." - ) - - # 4. Return fig, msg - return fig, msg diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_comparisons.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_comparisons.py deleted file mode 100644 index 076ebbbc2..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/throughput_comparisons.py +++ /dev/null @@ -1,565 +0,0 @@ -from dash import html, dcc -import plotly.graph_objects as go -import plotly.express as px -import pandas as pd -import copy - -import projects.matrix_benchmarking.visualizations.helpers.plotting.report as report - -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common - - -def register(): - BaselineComparisonsReport() - IntelligentRoutingComparisonsReport() - PDComparisonsReport() - - -def _generate_ttft_percentiles_plot(args): - """Generate TTFT percentiles plot in single chart""" - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - entry_name = entry.get_name(variables) - platform = getattr(entry.settings, "platform", "unknown") - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - # Use available TTFT fields with fallbacks for missing percentiles - ttft_p50 = getattr(benchmark, "ttft_p50", getattr(benchmark, "ttft_median", None)) - ttft_p90 = getattr(benchmark, "ttft_p90", getattr(benchmark, "ttft_p95", None)) - - data.append( - { - "Test Configuration": entry_name, - "Platform": platform, - "Concurrency": benchmark.request_concurrency, - "Strategy": benchmark.strategy, - "TTFT P10": getattr(benchmark, "ttft_p10", None), - "TTFT P25": getattr(benchmark, "ttft_p25", None), - "TTFT P50": ttft_p50, - "TTFT P75": getattr(benchmark, "ttft_p75", None), - "TTFT P90": ttft_p90, - } - ) - - if not data: - return html.P("No TTFT data available") - - df = pd.DataFrame(data) - df = df.sort_values(["Concurrency", "Test Configuration"]) - - # Create plot - use same color assignment as px.line() - fig = go.Figure() - configurations = sorted(df["Test Configuration"].unique()) - - # Create a dummy px.line figure to get the same color assignment - dummy_df = pd.DataFrame( - { - "Test Configuration": configurations, - "x": range(len(configurations)), - "y": range(len(configurations)), - } - ) - dummy_fig = px.line(dummy_df, x="x", y="y", color="Test Configuration") - color_map = {} - for trace in dummy_fig.data: - config_name = trace.name - color_map[config_name] = trace.line.color - - percentiles_config = [ - {"percentile": "P10", "line_style": {"width": 2, "dash": "longdash"}, "opacity": 0.6}, - {"percentile": "P25", "line_style": {"width": 2, "dash": "dot"}, "opacity": 0.7}, - {"percentile": "P50", "line_style": {"width": 4, "dash": "solid"}, "opacity": 1.0}, - {"percentile": "P75", "line_style": {"width": 3, "dash": "dash"}, "opacity": 0.9}, - {"percentile": "P90", "line_style": {"width": 2, "dash": "dashdot"}, "opacity": 0.8}, - ] - - for config_name in configurations: - config_df = df[df["Test Configuration"] == config_name].sort_values("Concurrency") - if len(config_df) == 0: - continue - - for perc_config in percentiles_config: - percentile = perc_config["percentile"] - column_name = f"TTFT {percentile}" - - # Skip if column doesn't exist or all values are None/NaN - if ( - column_name not in config_df.columns - or config_df[column_name].isna().all() - or config_df[column_name].isnull().all() - ): - continue - - trace_name = f"{config_name} - {percentile}" - base_color = color_map.get(config_name, px.colors.qualitative.Set1[0]) - - fig.add_trace( - go.Scatter( - x=config_df["Concurrency"], - y=config_df[column_name], - mode="lines+markers", - name=trace_name, - legendgroup=f"{config_name}", - marker=dict(size=6), - line=dict(color=base_color, **perc_config["line_style"]), - opacity=perc_config["opacity"], - hovertemplate=f"{config_name}
    TTFT {percentile}
    Concurrency: %{{x}}
    Value: %{{y:.1f}} ms
    ", - showlegend=True, - ) - ) - - fig.update_layout( - title="TTFT Percentiles vs Concurrency
    Lower is better • P10 (long dash), P25 (dotted), P50 (solid), P75 (dashed), P90 (dash-dot)", - showlegend=True, - hovermode="closest", - height=600, - xaxis_title="Concurrency", - yaxis_title="TTFT (ms)", - ) - fig.update_yaxes(rangemode="tozero") - - return dcc.Graph(figure=fig) - - -def _generate_itl_percentiles_plot(args): - """Generate ITL percentiles plot in single chart""" - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - # Generate DataFrame - data = [] - for entry in entries: - if not entry.results.guidellm_benchmarks: - continue - - entry_name = entry.get_name(variables) - platform = getattr(entry.settings, "platform", "unknown") - - for benchmark in entry.results.guidellm_benchmarks: - if benchmark.strategy == "throughput": - continue - - # Use available ITL fields with fallbacks for missing percentiles - itl_p50 = getattr(benchmark, "itl_p50", getattr(benchmark, "itl_median", None)) - itl_p90 = getattr(benchmark, "itl_p90", getattr(benchmark, "itl_p95", None)) - - data.append( - { - "Test Configuration": entry_name, - "Platform": platform, - "Concurrency": benchmark.request_concurrency, - "Strategy": benchmark.strategy, - "ITL P10": getattr(benchmark, "itl_p10", None), - "ITL P25": getattr(benchmark, "itl_p25", None), - "ITL P50": itl_p50, - "ITL P75": getattr(benchmark, "itl_p75", None), - "ITL P90": itl_p90, - } - ) - - if not data: - return html.P("No ITL data available") - - df = pd.DataFrame(data) - df = df.sort_values(["Concurrency", "Test Configuration"]) - - # Convert from seconds to milliseconds - for col in ["ITL P10", "ITL P25", "ITL P50", "ITL P75", "ITL P90"]: - if col in df.columns: - df[col] = df[col] * 1000 # Convert to ms - - # Create plot - use same color assignment as px.line() - fig = go.Figure() - configurations = sorted(df["Test Configuration"].unique()) - - # Create a dummy px.line figure to get the same color assignment - dummy_df = pd.DataFrame( - { - "Test Configuration": configurations, - "x": range(len(configurations)), - "y": range(len(configurations)), - } - ) - dummy_fig = px.line(dummy_df, x="x", y="y", color="Test Configuration") - color_map = {} - for trace in dummy_fig.data: - config_name = trace.name - color_map[config_name] = trace.line.color - - percentiles_config = [ - {"percentile": "P10", "line_style": {"width": 2, "dash": "longdash"}, "opacity": 0.6}, - {"percentile": "P25", "line_style": {"width": 2, "dash": "dot"}, "opacity": 0.7}, - {"percentile": "P50", "line_style": {"width": 4, "dash": "solid"}, "opacity": 1.0}, - {"percentile": "P75", "line_style": {"width": 3, "dash": "dash"}, "opacity": 0.9}, - {"percentile": "P90", "line_style": {"width": 2, "dash": "dashdot"}, "opacity": 0.8}, - ] - - for config_name in configurations: - config_df = df[df["Test Configuration"] == config_name].sort_values("Concurrency") - if len(config_df) == 0: - continue - - for perc_config in percentiles_config: - percentile = perc_config["percentile"] - column_name = f"ITL {percentile}" - - # Skip if column doesn't exist or all values are None/NaN - if ( - column_name not in config_df.columns - or config_df[column_name].isna().all() - or config_df[column_name].isnull().all() - ): - continue - - trace_name = f"{config_name} - {percentile}" - base_color = color_map.get(config_name, px.colors.qualitative.Set1[0]) - - fig.add_trace( - go.Scatter( - x=config_df["Concurrency"], - y=config_df[column_name], - mode="lines+markers", - name=trace_name, - legendgroup=f"{config_name}", - marker=dict(size=6), - line=dict(color=base_color, **perc_config["line_style"]), - opacity=perc_config["opacity"], - hovertemplate=f"{config_name}
    ITL {percentile}
    Concurrency: %{{x}}
    Value: %{{y:.1f}} ms
    ", - showlegend=True, - ) - ) - - fig.update_layout( - title="ITL Percentiles vs Concurrency
    Lower is better • P10 (long dash), P25 (dotted), P50 (solid), P75 (dashed), P90 (dash-dot)", - showlegend=True, - hovermode="closest", - height=600, - xaxis_title="Concurrency", - yaxis_title="ITL (ms)", - ) - fig.update_yaxes(rangemode="tozero") - - return dcc.Graph(figure=fig) - - -def _generate_throughput_plots(args): - """ - Generate throughput plots with five tabs: Throughput (mean), TTFT P50, TTFT Percentiles, ITL P50, ITL Percentiles - - Args: - args: Plot arguments (potentially filtered for specific model/load_shape) - - Returns: - List of HTML elements containing the tabbed plots - """ - # Tab 1: Throughput (Mean) - throughput_content = [] - throughput_content.append(html.H4("🚀 Token Throughput vs Concurrency (Mean)")) - throughput_content.append( - html.P("Token generation throughput scaling analysis using mean values.") - ) - throughput_content += report.Plot_and_Text("Guidellm Tokens vs Concurrency", args) - - # Tab 2: TTFT P50 - ttft_p50_content = [] - ttft_p50_content.append(html.H4("⏱️ Time To First Token (P50)")) - ttft_p50_content.append(html.P("Median time to first token analysis.")) - ttft_p50_content += report.Plot_and_Text("Guidellm TTFT Analysis", args) - - # Tab 3: TTFT Percentiles - ttft_percentiles_content = [] - ttft_percentiles_content.append(html.H4("📊 TTFT Percentiles")) - ttft_percentiles_content.append( - html.P("Complete TTFT percentile distribution analysis (P10, P25, P50, P75, P90).") - ) - ttft_percentiles_content.append(_generate_ttft_percentiles_plot(args)) - - # Tab 4: ITL P50 - itl_p50_content = [] - itl_p50_content.append(html.H4("🔄 Inter-Token Latency (P50)")) - itl_p50_content.append(html.P("Median inter-token latency analysis.")) - itl_p50_content += report.Plot_and_Text("Guidellm ITL Analysis", args) - - # Tab 5: ITL Percentiles - itl_percentiles_content = [] - itl_percentiles_content.append(html.H4("📊 ITL Percentiles")) - itl_percentiles_content.append( - html.P("Complete ITL percentile distribution analysis (P10, P25, P50, P75, P90).") - ) - itl_percentiles_content.append(_generate_itl_percentiles_plot(args)) - - # Create tabs - tabs = dcc.Tabs( - id="throughput-tabs", - value="throughput", - children=[ - dcc.Tab( - label="🚀 Throughput", - value="throughput", - children=[html.Div(throughput_content, style={"padding": "20px"})], - ), - dcc.Tab( - label="⏱️ TTFT P50", - value="ttft-p50", - children=[html.Div(ttft_p50_content, style={"padding": "20px"})], - ), - dcc.Tab( - label="📊 TTFT Percentiles", - value="ttft-percentiles", - children=[html.Div(ttft_percentiles_content, style={"padding": "20px"})], - ), - dcc.Tab( - label="🔄 ITL P50", - value="itl-p50", - children=[html.Div(itl_p50_content, style={"padding": "20px"})], - ), - dcc.Tab( - label="📊 ITL Percentiles", - value="itl-percentiles", - children=[html.Div(itl_percentiles_content, style={"padding": "20px"})], - ), - ], - ) - - return [tabs] - - -class BaselineComparisonsReport: - def __init__(self): - self.name = "report: Baseline Comparisons" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate baseline comparison report for simple flavor configurations - """ - - ordered_vars, settings, setting_lists, variables, cfg = args - - header = [] - header.append(html.H2("📊 Baseline Comparisons")) - header.append(html.Br()) - - header.append( - html.P( - [ - "Baseline performance analysis using simple flavor configuration, ", - "comparing across different models and load shapes.", - ] - ) - ) - header.append(html.Br()) - - args = report.set_config(dict(markers_by="platform"), args) - - def filter_flavors(setting_lists, flavor_filter): - """Filter flavors from setting_lists based on provided filter function""" - updated_setting_lists = [] - for setting_list in setting_lists: - if setting_list and setting_list[0][0] == "flavor": - # Apply the filter function to flavors - filtered_flavors = [(k, v) for k, v in setting_list if flavor_filter(v)] - if filtered_flavors: - updated_setting_lists.append(filtered_flavors) - else: - updated_setting_lists.append(setting_list) - setting_lists[:] = updated_setting_lists - - # Get simple flavors - simple_flavors = [ - f for f in common.Matrix.settings.get("flavor", []) if f.startswith("simple") - ] - - for load_shape in common.Matrix.settings.get("load_shape", []): - # Skip multiturn load shape for baseline - if load_shape == "Multiturn": - continue - - header.append(html.H3(f"📊 Load Shape: {load_shape}")) - - for flavor in simple_flavors: - header.append(html.H4(f"🔧 {flavor}")) - if flavor == "simple-tp4-x4": - header.append(html.I(f"Skipped, not relevant for baseline.")) - continue - - # Set llama3.3-70b model and specific simple flavor - baseline_settings = { - "model": "llama3.3-70b", - "load_shape": load_shape, - "flavor": flavor, - } - baseline_args = report.set_settings(baseline_settings, args) - - header += _generate_throughput_plots(baseline_args) - - return None, header - - -class IntelligentRoutingComparisonsReport: - def __init__(self): - self.name = "report: Intelligent-routing Comparisons" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate intelligent routing comparison report - """ - - ordered_vars, settings, setting_lists, variables, cfg = args - - header = [] - header.append(html.H2("🧠 Intelligent Routing Comparisons")) - header.append(html.Br()) - - header.append( - html.P( - [ - "Analysis of intelligent routing performance using llama3.3-70b model, ", - "comparing routing-enabled configurations across different load shapes.", - ] - ) - ) - header.append(html.Br()) - - args = report.set_config(dict(markers_by="platform"), args) - - def filter_flavors(setting_lists, flavor_filter): - """Filter flavors from setting_lists based on provided filter function""" - updated_setting_lists = [] - for setting_list in setting_lists: - if setting_list and setting_list[0][0] == "flavor": - # Apply the filter function to flavors - filtered_flavors = [(k, v) for k, v in setting_list if flavor_filter(v)] - if filtered_flavors: - updated_setting_lists.append(filtered_flavors) - else: - updated_setting_lists.append(setting_list) - setting_lists[:] = updated_setting_lists - - for with_simple in False, True: - if with_simple: - header.append(html.H2("Intelligent Routing VS native")) - else: - header.append(html.H2("Intelligent Routing")) - - for load_shape in common.Matrix.settings.get("load_shape", []): - header.append(html.H3(f"📊 Load Shape: {load_shape}")) - - # Set llama3.3-70b and remove simple flavor - ir_settings = {"model": "llama3.3-70b", "load_shape": load_shape} - ir_args = report.set_settings(ir_settings, args) - - # Filter out simple flavor from the remaining args if it exists - ordered_vars, settings, setting_lists, variables_filtered, cfg = ir_args - - def include_ir_flavor(v): - if v.startswith("pd-"): - return False - if not with_simple and v.startswith("simple"): - return False - if v.startswith("simple") and not v.endswith("tp2-x4"): - return False - - return True - - # Remove simple and pd- flavors - filter_flavors(setting_lists, include_ir_flavor) - ir_args_final = (ordered_vars, settings, setting_lists, variables_filtered, cfg) - - header += _generate_throughput_plots(ir_args_final) - - return None, header - - -class PDComparisonsReport: - def __init__(self): - self.name = "report: P-D Comparisons" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate P/D disaggregation comparison report - """ - - ordered_vars, settings, setting_lists, variables, cfg = args - - header = [] - header.append(html.H2("🔄 P/D Disaggregation Comparisons")) - header.append(html.Br()) - - header.append( - html.P( - [ - "Prefill/Decode disaggregation analysis using gpt-oss-120b model, ", - "comparing disaggregated configurations across different load shapes.", - ] - ) - ) - header.append(html.Br()) - - args = report.set_config(dict(markers_by="platform"), args) - - def filter_flavors(setting_lists, flavor_filter): - """Filter flavors from setting_lists based on provided filter function""" - updated_setting_lists = [] - for setting_list in setting_lists: - if setting_list and setting_list[0][0] == "flavor": - # Apply the filter function to flavors - filtered_flavors = [(k, v) for k, v in setting_list if flavor_filter(v)] - if filtered_flavors: - updated_setting_lists.append(filtered_flavors) - else: - updated_setting_lists.append(setting_list) - setting_lists[:] = updated_setting_lists - - for load_shape in common.Matrix.settings.get("load_shape", []): - header.append(html.H3(f"📊 Load Shape: {load_shape}")) - - # Set gpt-oss-120b model - pd_settings = {"model": "gpt-oss-120b", "load_shape": load_shape} - pd_args = report.set_settings(pd_settings, args) - - # Filter to show only pd- flavors - ordered_vars, settings, setting_lists, variables_filtered, cfg = pd_args - - def include_pd_flavor(v): - if "(eth)" in v: - return False - # if "(sched v4)" in v: return False - if not (v.startswith("pd-") or v.endswith("-tp4-x4")): - return False - return True - - filter_flavors(setting_lists, include_pd_flavor) - pd_args_final = (ordered_vars, settings, setting_lists, variables_filtered, cfg) - - header += _generate_throughput_plots(pd_args_final) - - return None, header diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/utils.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/utils.py deleted file mode 100644 index e775a3991..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/utils.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Utility functions for LLM-D visualization plotting modules -""" - -import logging -from dash import html -import matrix_benchmarking.common as common - - -def check_prometheus_data_availability(metric_type, report_name): - """ - Check if Prometheus metrics data is available and return appropriate response - - Args: - metric_type: Type of metrics to check ("sutest" for main Prometheus, "uwm" for User Workload Monitoring) - report_name: Human-readable name of the report for logging - - Returns: - tuple: (has_data: bool, error_elements: list) - - has_data: True if metrics data is available - - error_elements: HTML elements to display if no data (empty list if data available) - """ - entries = list(common.Matrix.all_records(None, None)) - - # Check if any entries have the requested metrics data - has_metrics_data = False - for entry in entries: - try: - if ( - hasattr(entry.results, "metrics") - and entry.results.metrics - and entry.results.metrics.get(metric_type) - ): - has_metrics_data = True - break - except Exception: - continue - - if not has_metrics_data: - logging.info( - f"No {metric_type} Prometheus metrics data found - skipping {report_name} generation" - ) - - # Generate appropriate error message based on metric type - if metric_type == "uwm": - error_elements = [ - html.P("⚠️ No Prometheus User Workload Monitoring metrics data available."), - html.P("This report requires UWM metrics to be captured during the test run."), - html.P( - "To enable metrics collection, set tests.capture_prom_uwm: true in the configuration." - ), - ] - else: # sutest/main prometheus - error_elements = [ - html.P("⚠️ No Prometheus cluster metrics data available."), - html.P( - "This report requires cluster-level Prometheus metrics to be captured during the test run." - ), - html.P( - "To enable metrics collection, set tests.capture_prom: true in the configuration." - ), - ] - - return False, error_elements - - return True, [] diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/vllm_metrics.py b/projects/llm_d_legacy/visualizations/llmd_inference/plotting/vllm_metrics.py deleted file mode 100644 index 01fb2425b..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/plotting/vllm_metrics.py +++ /dev/null @@ -1,369 +0,0 @@ -from dash import html - -import projects.matrix_benchmarking.visualizations.helpers.plotting.prom as plotting_prom -import projects.matrix_benchmarking.visualizations.helpers.plotting.report as report - -import matrix_benchmarking.plotting.table_stats as table_stats -import matrix_benchmarking.common as common - -from .utils import check_prometheus_data_availability - - -def register(): - """Register all VLLM-specific Prometheus metrics as individual plots""" - - # Get metrics function for UWM data - def get_uwm_metrics(entry, metric): - try: - return entry.results.metrics["uwm"][metric] - except KeyError: - return [] - - # ========================================================================= - # 🚀 END-TO-END LATENCY METRICS - # ========================================================================= - - # E2E Request Latency - Average latency per request - plotting_prom.Plot( - ["vllm_e2e_latency_average"], - "Prom: VLLM E2E Request Latency (Average)", - title="VLLM End-to-End Request Latency (Average)", - y_title="Latency (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # E2E Request Rate - Requests per second - plotting_prom.Plot( - [{"vllm_e2e_request_rate": "rate(vllm_e2e_latency_seconds_count[5m])"}], - "Prom: VLLM E2E Request Rate", - title="VLLM End-to-End Request Rate", - y_title="Requests/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # ========================================================================= - # ⚡ TIME TO FIRST TOKEN (TTFT) METRICS - # ========================================================================= - - # TTFT Average latency - plotting_prom.Plot( - ["vllm_ttft_average"], - "Prom: VLLM Time to First Token (Average)", - title="VLLM Time to First Token (Average)", - y_title="TTFT (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # TTFT Request Rate - plotting_prom.Plot( - ["vllm_ttft_rate"], - "Prom: VLLM TTFT Request Rate", - title="VLLM TTFT Request Rate", - y_title="Requests/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # ========================================================================= - # 🔄 INTER-TOKEN LATENCY METRICS - # ========================================================================= - - # Inter-token Average latency - plotting_prom.Plot( - ["vllm_inter_token_average"], - "Prom: VLLM Inter-Token Latency (Average)", - title="VLLM Inter-Token Latency (Average)", - y_title="Latency (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Inter-token Request Rate - plotting_prom.Plot( - ["vllm_inter_token_rate"], - "Prom: VLLM Inter-Token Request Rate", - title="VLLM Inter-Token Request Rate", - y_title="Requests/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # ========================================================================= - # 🔧 REQUEST PROCESSING PHASE METRICS - # ========================================================================= - - # Prefill Phase - Average time per request - plotting_prom.Plot( - ["vllm_prefill_average"], - "Prom: VLLM Request Prefill Time (Average)", - title="VLLM Request Prefill Time (Average)", - y_title="Time (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Prefill Phase - Rate of time spent - plotting_prom.Plot( - ["vllm_prefill_rate"], - "Prom: VLLM Request Prefill Time Rate", - title="VLLM Request Prefill Time Rate", - y_title="Seconds/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Decode Phase - Average time per request - plotting_prom.Plot( - ["vllm_decode_average"], - "Prom: VLLM Request Decode Time (Average)", - title="VLLM Request Decode Time (Average)", - y_title="Time (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Decode Phase - Rate of time spent - plotting_prom.Plot( - ["vllm_decode_rate"], - "Prom: VLLM Request Decode Time Rate", - title="VLLM Request Decode Time Rate", - y_title="Seconds/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Queue Time - Average time per request - plotting_prom.Plot( - ["vllm_queue_average"], - "Prom: VLLM Request Queue Time (Average)", - title="VLLM Request Queue Time (Average)", - y_title="Time (seconds)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Queue Time - Rate of time spent - plotting_prom.Plot( - ["vllm_queue_rate"], - "Prom: VLLM Request Queue Time Rate", - title="VLLM Request Queue Time Rate", - y_title="Seconds/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # ========================================================================= - # 🔤 TOKEN THROUGHPUT METRICS - # ========================================================================= - - # Token throughput rates - plotting_prom.Plot( - ["vllm_prompt_tokens_rate"], - "Prom: VLLM Prompt Tokens Rate", - title="VLLM Prompt Tokens Rate", - y_title="Tokens/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - plotting_prom.Plot( - ["vllm_generation_tokens_rate"], - "Prom: VLLM Generation Tokens Rate", - title="VLLM Generation Tokens Rate", - y_title="Tokens/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Combined token throughput - plotting_prom.Plot( - ["vllm_total_tokens_rate"], - "Prom: VLLM Total Tokens Rate", - title="VLLM Total Tokens Rate (Prompt + Generation)", - y_title="Tokens/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Average max generation tokens per request - plotting_prom.Plot( - ["vllm_avg_max_gen_tokens"], - "Prom: VLLM Avg Max Generation Tokens", - title="VLLM Average Max Generation Tokens per Request", - y_title="Tokens", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # ========================================================================= - # 📊 REQUEST QUEUE STATE AND CAPACITY - # ========================================================================= - - # Current queue state (gauge metrics - no transformation needed) - plotting_prom.Plot( - ["vllm_num_requests_running"], - "Prom: VLLM Running Requests", - title="VLLM Number of Running Requests", - y_title="Request Count", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - plotting_prom.Plot( - ["vllm_num_requests_waiting"], - "Prom: VLLM Waiting Requests", - title="VLLM Number of Waiting Requests", - y_title="Request Count", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - plotting_prom.Plot( - ["vllm_kv_cache_usage_perc"], - "Prom: VLLM KV Cache Usage", - title="VLLM KV Cache Usage Percentage", - y_title="Usage (%)", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Request success rate - plotting_prom.Plot( - ["vllm_request_success_rate"], - "Prom: VLLM Request Success Rate", - title="VLLM Request Success Rate", - y_title="Successful Requests/sec", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - # Request success increase (over time window) - plotting_prom.Plot( - ["vllm_request_success_increase"], - "Prom: VLLM Request Success Increase", - title="VLLM Request Success (5m increase)", - y_title="Successful Requests", - get_metrics=get_uwm_metrics, - as_timestamp=True, - ) - - -class VLLMMetricsReport: - def __init__(self): - self.name = "report: VLLM Metrics Analysis" - self.id_name = self.name.lower().replace(" ", "_").replace("-", "_") - self.no_graph = True - self.is_report = True - - table_stats.TableStats._register_stat(self) - common.Matrix.settings["stats"].add(self.name) - - def do_plot(self, *args): - """ - Generate comprehensive VLLM metrics analysis report - """ - ordered_vars, settings, setting_lists, variables, cfg = args - entries = list(common.Matrix.all_records(settings, setting_lists)) - - header = [] - header.append(html.H2("🔥 VLLM Performance Metrics Analysis")) - header.append( - html.P( - "Comprehensive analysis of VLLM inference server metrics from User Workload Monitoring" - ) - ) - header.append(html.Br()) - - if not entries: - header.append(html.P("No test entries found.")) - return None, header - - # Check if Prometheus UWM metrics data is available - has_data, error_elements = check_prometheus_data_availability("uwm", "VLLM metrics report") - if not has_data: - header.extend(error_elements) - return None, header - - # End-to-End Latency Section - header.append(html.H3("🚀 End-to-End Request Latency")) - header.append( - html.P("Overall request processing time and throughput from client perspective") - ) - - header += report.Plot_and_Text("Prom: VLLM E2E Request Latency (Average)", args) - header += report.Plot_and_Text("Prom: VLLM E2E Request Rate", args) - - header.append(html.Br()) - - # Time to First Token Section - header.append(html.H3("⚡ Time to First Token (TTFT)")) - header.append( - html.P( - "Critical metric for perceived responsiveness - time until first token is generated" - ) - ) - - header += report.Plot_and_Text("Prom: VLLM Time to First Token (Average)", args) - header += report.Plot_and_Text("Prom: VLLM TTFT Request Rate", args) - - header.append(html.Br()) - - # Inter-Token Latency Section - header.append(html.H3("🔄 Inter-Token Latency")) - header.append(html.P("Streaming quality - time between generated tokens")) - - header += report.Plot_and_Text("Prom: VLLM Inter-Token Latency (Average)", args) - header += report.Plot_and_Text("Prom: VLLM Inter-Token Request Rate", args) - - header.append(html.Br()) - - # Request Processing Phases Section - header.append(html.H3("🔧 Request Processing Phases")) - header.append( - html.P("Detailed breakdown of request processing: prefill, decode, and queue times") - ) - - header.append(html.H4("Prefill Phase")) - header += report.Plot_and_Text("Prom: VLLM Request Prefill Time (Average)", args) - header += report.Plot_and_Text("Prom: VLLM Request Prefill Time Rate", args) - - header.append(html.H4("Decode Phase")) - header += report.Plot_and_Text("Prom: VLLM Request Decode Time (Average)", args) - header += report.Plot_and_Text("Prom: VLLM Request Decode Time Rate", args) - - header.append(html.H4("Queue Time")) - header += report.Plot_and_Text("Prom: VLLM Request Queue Time (Average)", args) - header += report.Plot_and_Text("Prom: VLLM Request Queue Time Rate", args) - - header.append(html.Br()) - - # Token Throughput Section - header.append(html.H3("🔤 Token Throughput")) - header.append(html.P("Token generation rates and processing statistics")) - - header += report.Plot_and_Text("Prom: VLLM Prompt Tokens Rate", args) - header += report.Plot_and_Text("Prom: VLLM Generation Tokens Rate", args) - header += report.Plot_and_Text("Prom: VLLM Total Tokens Rate", args) - header += report.Plot_and_Text("Prom: VLLM Avg Max Generation Tokens", args) - - header.append(html.Br()) - - # Request Queue State Section - header.append(html.H3("📊 Request Queue State & Success Metrics")) - header.append(html.P("Server capacity utilization, queue state, and success rates")) - - header += report.Plot_and_Text("Prom: VLLM Running Requests", args) - header += report.Plot_and_Text("Prom: VLLM Waiting Requests", args) - header += report.Plot_and_Text("Prom: VLLM KV Cache Usage", args) - header += report.Plot_and_Text("Prom: VLLM Request Success Rate", args) - header += report.Plot_and_Text("Prom: VLLM Request Success Increase", args) - - return None, header - - -def register_report(): - """Register the VLLM metrics report""" - VLLMMetricsReport() diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/requirements.txt b/projects/llm_d_legacy/visualizations/llmd_inference/requirements.txt deleted file mode 100644 index 9ddc4991c..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -matplotlib>=3.5.0 -pandas>=1.3.0 -numpy>=1.20.0 -plotly>=5.0.0 \ No newline at end of file diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/store/__init__.py b/projects/llm_d_legacy/visualizations/llmd_inference/store/__init__.py deleted file mode 100644 index 69c2deabc..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/store/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -import pathlib -import logging -import types -import os -import json - -import matrix_benchmarking.store as store -import matrix_benchmarking.store.simple as store_simple - -import projects.matrix_benchmarking.visualizations.helpers.store as helpers_store - -from . import parsers -from . import lts_parser -from ..models import lts as models_lts -from ..models import kpi as models_kpi - -CACHE_FILENAME = "cache.pickle" -IMPORTANT_FILES = parsers.IMPORTANT_FILES - - -class Store(helpers_store.BaseStore): - def prepare_for_pickle(self, results): - pass - - def prepare_after_pickle(self, results): - pass - - -local_store = Store( - cache_filename=CACHE_FILENAME, - important_files=IMPORTANT_FILES, - artifact_dirnames=parsers.artifact_dirnames, - artifact_paths=parsers.artifact_paths, - parse_always=parsers.parse_always, - parse_once=parsers.parse_once, - lts_payload_model=models_lts.Payload, - generate_lts_payload=lts_parser.generate_lts_payload, - models_kpis=models_kpi.KPIs, - get_kpi_labels=lts_parser.get_kpi_labels, -) - -parsers.register_important_file = local_store.register_important_file -build_lts_payloads = local_store.build_lts_payloads -is_mandatory_file = local_store.is_mandatory_file -is_cache_file = local_store.is_cache_file -is_important_file = local_store.is_important_file - - -def _rewrite_settings(settings_dict, results=None, is_lts=None): - return settings_dict - - -# delegate the parsing to the simple_store -store.register_custom_rewrite_settings(_rewrite_settings) -store_simple.register_custom_parse_results(local_store.parse_directory) diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/store/lts_parser.py b/projects/llm_d_legacy/visualizations/llmd_inference/store/lts_parser.py deleted file mode 100644 index 74fc8add7..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/store/lts_parser.py +++ /dev/null @@ -1,96 +0,0 @@ -import types -import pathlib -import logging -import datetime -import os -import uuid - -import matrix_benchmarking.models as matbench_models -import matrix_benchmarking.parsing as matbench_parsing - -from ..models.lts import ( - LTS_SCHEMA_VERSION, - KPI_SETTINGS_VERSION, - Settings, - Metadata, - Results, - Payload, -) -from .parsers import parse_guidellm_benchmark_log - - -def generate_lts_payload(results, import_settings): - lts_payload = types.SimpleNamespace() - - lts_payload.metadata = generate_lts_metadata(results, import_settings) - lts_payload.results = generate_lts_results(results) - # lts_payload.kpis is generated in the helper store - - return lts_payload - - -def get_kpi_labels(lts_payload): - kpi_labels = dict(lts_payload.metadata.settings.__dict__) - return kpi_labels - - -def generate_lts_metadata(results, import_settings): - lts_metadata = types.SimpleNamespace() - - # Generate settings - settings_data = { - "kpi_settings_version": KPI_SETTINGS_VERSION, - "test_name": "llm-d-inference", - "namespace": "llm-d-project", # Default, could be read from config - "guidellm_enabled": len(results.guidellm_benchmarks) > 0, - } - - # Add any additional settings from import_settings - if import_settings: - for key, value in import_settings.items(): - if key not in settings_data: - settings_data[key] = value - - lts_metadata.settings = Settings(**settings_data) - - # Generate metadata - lts_metadata.lts_schema_version = LTS_SCHEMA_VERSION - lts_metadata.presets = [] - - # Generate timestamps - use current time if not available - start_time = datetime.datetime.now() - end_time = start_time - - if hasattr(results, "test_timestamp") and results.test_timestamp: - start_time = results.test_timestamp - end_time = results.test_timestamp - - lts_metadata.start = start_time - lts_metadata.end = end_time - lts_metadata.test_uuid = uuid.uuid4() - lts_metadata.urls = None - lts_metadata.exit_code = 0 if results.test_success else 1 - - return lts_metadata - - -def generate_lts_results(results): - results_lts = types.SimpleNamespace() - - results_lts.test_name = getattr(results, "test_name", "llm-d-inference") - results_lts.test_success = getattr(results, "test_success", True) - results_lts.test_failure_reason = getattr(results, "test_failure_reason", None) - - # Copy benchmark results - results_lts.guidellm_benchmarks = results.guidellm_benchmarks - - # Copy data paths only if they have values - guidellm_log_path = getattr(results, "guidellm_log_path", None) - if guidellm_log_path: - results_lts.guidellm_log_path = guidellm_log_path - - prometheus_path = getattr(results, "prometheus_path", None) - if prometheus_path: - results_lts.prometheus_path = prometheus_path - - return results_lts diff --git a/projects/llm_d_legacy/visualizations/llmd_inference/store/parsers.py b/projects/llm_d_legacy/visualizations/llmd_inference/store/parsers.py deleted file mode 100644 index 6fe4310ef..000000000 --- a/projects/llm_d_legacy/visualizations/llmd_inference/store/parsers.py +++ /dev/null @@ -1,806 +0,0 @@ -import pathlib -import logging -import datetime -import re -import json -import csv -import types -import glob - -import projects.matrix_benchmarking.visualizations.helpers.store.prom as helper_prom_store -import projects.matrix_benchmarking.visualizations.helpers.store.parsers as helpers_store_parsers - -from ..models import GuidellmBenchmark - -register_important_file = None # will be when importing store/__init__.py - -# Define artifact directory patterns -artifact_dirnames = types.SimpleNamespace() -artifact_dirnames.GUIDELLM_BENCHMARK_DIR = "*__llmd__run_guidellm_benchmark" -artifact_dirnames.PROMETHEUS_DUMP_DIR = ( - "*__cluster__dump_prometheus_dbs/*__cluster__dump_prometheus_db" -) -artifact_dirnames.PROMETHEUS_UWM_DUMP_DIR = ( - "*__cluster__dump_prometheus_dbs/*__cluster__dump_prometheus_db_uwm" -) -artifact_dirnames.LLMISVC_CAPTURE_DIR = "*__llmd__capture_isvc_state" - -artifact_paths = types.SimpleNamespace() # will be dynamically populated - -# Define important files we want to parse -IMPORTANT_FILES = [ - "exit_code", - "settings.yaml", - # Note: GuideLLM benchmark files are now handled dynamically in parse_once() - # to support multiple benchmark directories (multi-rate scenarios) - f"{artifact_dirnames.PROMETHEUS_DUMP_DIR}/prometheus.t*", - f"{artifact_dirnames.PROMETHEUS_UWM_DUMP_DIR}/prometheus.t*", - f"{artifact_dirnames.GUIDELLM_BENCHMARK_DIR}/artifacts/results/benchmarks.json", - f"{artifact_dirnames.GUIDELLM_BENCHMARK_DIR}/artifacts/guidellm_benchmark_job.logs", - f"{artifact_dirnames.LLMISVC_CAPTURE_DIR}/artifacts/llminferenceservice.json", -] - - -def parse_always(results, dirname, import_settings): - """Parsed even when reloading from the cache file""" - results.from_local_env = helpers_store_parsers.parse_local_env(dirname) - - -def find_guidellm_benchmark_directories(dirname): - """Find all guidellm benchmark directories, including multi-rate directories""" - # Look for all directories matching the guidellm benchmark pattern - # This includes both single directory and multi-rate directories - pattern = "*__llmd__run_guidellm_benchmark*" - guidellm_dirs = [] - - for path in glob.glob(str(dirname / pattern)): - path_obj = pathlib.Path(path) - if path_obj.is_dir(): - guidellm_dirs.append(path_obj) - - # Sort to ensure consistent ordering - return sorted(guidellm_dirs) - - -def parse_once(results, dirname): - """Parse the benchmark log files once""" - - # Parse test configuration and environment for PROW URLs - results.from_env = helpers_store_parsers.parse_env( - dirname, None, artifact_paths.LLMISVC_CAPTURE_DIR - ) - - # Parse guidellm benchmarks - support multiple benchmark directories - results.guidellm_benchmarks = [] - results.guidellm_configuration = None - guidellm_directories = find_guidellm_benchmark_directories(dirname) - - if guidellm_directories: - for guidellm_dir in guidellm_directories: - # Check for JSON file first, fallback to log file - json_file_path = guidellm_dir / "artifacts" / "results" / "benchmarks.json" - log_file_path = guidellm_dir / "artifacts" / "guidellm_benchmark_job.logs" - - if json_file_path.exists(): - benchmarks = parse_guidellm_benchmark_json( - dirname, json_file_path.relative_to(dirname) - ) - results.guidellm_benchmarks.extend(benchmarks) - logging.info( - f"Parsed {len(benchmarks)} guidellm benchmarks from JSON: {json_file_path}" - ) - - # Parse configuration from the first JSON file found - if results.guidellm_configuration is None: - results.guidellm_configuration = _parse_guidellm_config( - dirname, json_file_path.relative_to(dirname) - ) - - elif log_file_path.exists(): - raise RuntimeError("Don't want to use log-file parsing (hardcoded)") - benchmarks = parse_guidellm_benchmark_log( - dirname, log_file_path.relative_to(dirname) - ) - results.guidellm_benchmarks.extend(benchmarks) - logging.info( - f"Parsed {len(benchmarks)} guidellm benchmarks from log: {log_file_path}" - ) - else: - logging.warning(f"Neither JSON nor log file found in {guidellm_dir / 'artifacts'}") - - logging.info(f"Total parsed guidellm benchmarks: {len(results.guidellm_benchmarks)}") - else: - logging.warning("No guidellm benchmark directories found") - - # Parse test metadata - exit_code_path = "exit_code" - if exit_code_path and (dirname / exit_code_path).exists(): - with open(register_important_file(dirname, exit_code_path)) as f: - exit_code = f.read().strip() - results.test_success = exit_code == "0" - logging.info(f"Test exit code: {exit_code}, success: {results.test_success}") - else: - results.test_success = None - logging.warning(f"Exit code file not found at {exit_code_path}") - - # Parse LLMISVC configuration - results.llmisvc_config = _parse_llmisvc_config(dirname) - - # Parse Prometheus UWM metrics - results.metrics = _extract_metrics(dirname) - - -def parse_guidellm_benchmark_log(dirname, log_file_path: pathlib.Path) -> list[GuidellmBenchmark]: - """Parse Guidellm benchmark log file and extract metrics for each strategy""" - - if not (dirname / log_file_path).exists(): - logging.warning(f"Guidellm benchmark log not found: {log_file_path}") - return [] - - with open(register_important_file(dirname, log_file_path)) as f: - content = f.read() - - logging.debug(f"Parsing GuideLLM log file: {log_file_path} (length: {len(content)} chars)") - - # Check if log contains the expected output format - if "Run Summary Info" not in content: - logging.warning( - f"GuideLLM log appears to be incomplete or malformed - no 'Run Summary Info' section found in {log_file_path}" - ) - return [] - - benchmarks = [] - - # Parse the Run Summary Info table for basic info - summary_match = re.search( - r"ℹ Run Summary Info.*?\n(.*?)(?=ℹ Text Metrics Statistics|\Z)", content, re.DOTALL - ) - if not summary_match: - logging.warning("Could not find Run Summary Info table in Guidellm log") - return benchmarks - - summary_table = summary_match.group(1) - - # Parse the Request Latency Statistics table - latency_match = re.search( - r"ℹ Request Latency Statistics.*?\n(.*?)(?=ℹ Server Throughput Statistics|\Z)", - content, - re.DOTALL, - ) - if not latency_match: - logging.warning("Could not find Request Latency Statistics table in Guidellm log") - return benchmarks - - latency_table = latency_match.group(1) - - # Parse the Server Throughput Statistics table - throughput_match = re.search( - r"ℹ Server Throughput Statistics.*?\n(.*?)(?=\n\n|\Z)", content, re.DOTALL - ) - if not throughput_match: - logging.warning("Could not find Server Throughput Statistics table in Guidellm log") - return benchmarks - - throughput_table = throughput_match.group(1) - - # Extract data rows (skip headers and separators) - def extract_data_rows(table_text): - lines = [line.strip() for line in table_text.split("\n") if line.strip()] - data_rows = [] - for line in lines: - if ( - "|" in line - and not line.startswith("|===") - and "Benchmark" not in line - and "Strategy" not in line - and "------" not in line - ): - data_rows.append(line) - return data_rows - - summary_rows = extract_data_rows(summary_table) - latency_rows = extract_data_rows(latency_table) - throughput_rows = extract_data_rows(throughput_table) - - # Parse each benchmark strategy - for i, summary_row in enumerate(summary_rows): - if i >= len(latency_rows) or i >= len(throughput_rows): - break - - try: - # Parse summary row: | concurrent | 17:10:09 | 17:11:09 | 60.0 | 0.0 | 0.0 | 6809.0 | 0.0 | 0.0 | 3048.0 | 0.0 | 0.0 | - summary_parts = [part.strip() for part in summary_row.split("|") if part.strip()] - if len(summary_parts) < 12: - continue - - strategy = summary_parts[0] - duration = float(summary_parts[3]) if summary_parts[3] else 60.0 - warmup_time = float(summary_parts[4]) if summary_parts[4] else 0.0 - cooldown_time = float(summary_parts[5]) if summary_parts[5] else 0.0 - input_tokens_comp = float(summary_parts[6]) if summary_parts[6] else 0.0 - output_tokens_comp = float(summary_parts[9]) if summary_parts[9] else 0.0 - - # Parse latency row: | concurrent | 68.6 | 68.6 | 1366.9 | 1366.9 | 22.1 | 22.1 | 22.5 | 22.5 | - latency_parts = [part.strip() for part in latency_rows[i].split("|") if part.strip()] - if len(latency_parts) < 9: - continue - - request_latency_median = float(latency_parts[1]) if latency_parts[1] else 0.0 - request_latency_p95 = float(latency_parts[2]) if latency_parts[2] else 0.0 - ttft_median = float(latency_parts[3]) if latency_parts[3] else 0.0 - ttft_p95 = float(latency_parts[4]) if latency_parts[4] else 0.0 - itl_median = float(latency_parts[5]) if latency_parts[5] else 0.0 - itl_p95 = float(latency_parts[6]) if latency_parts[6] else 0.0 - tpot_median = float(latency_parts[7]) if latency_parts[7] else 0.0 - tpot_p95 = float(latency_parts[8]) if latency_parts[8] else 0.0 - - # Parse throughput row: | concurrent | 1.0 | 1.0 | 0.0 | 0.0 | 45.3 | 146.5 | - throughput_parts = [ - part.strip() for part in throughput_rows[i].split("|") if part.strip() - ] - if len(throughput_parts) < 7: - continue - - concurrency_median = float(throughput_parts[1]) if throughput_parts[1] else 0.0 - concurrency_mean = float(throughput_parts[2]) if throughput_parts[2] else 0.0 - request_rate = float(throughput_parts[3]) if throughput_parts[3] else 0.0 - input_tokens_per_second = float(throughput_parts[4]) if throughput_parts[4] else 0.0 - output_tokens_per_second = float(throughput_parts[5]) if throughput_parts[5] else 0.0 - total_tokens_per_second = float(throughput_parts[6]) if throughput_parts[6] else 0.0 - - # Calculate requests completed during the run - completed_requests = int(request_rate * duration) if request_rate > 0 else 0 - - # Calculate tokens per request - input_tokens_per_request = ( - (input_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - output_tokens_per_request = ( - (output_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - total_tokens_per_request = ( - (total_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - - benchmark = GuidellmBenchmark( - strategy=strategy, - duration=duration, - warmup_time=warmup_time, - cooldown_time=cooldown_time, - # Request metrics - request_rate=request_rate, - request_concurrency=concurrency_mean, - completed_requests=completed_requests, - failed_requests=0, # Not available in current format - # Token metrics per request - input_tokens_per_request=input_tokens_per_request, - output_tokens_per_request=output_tokens_per_request, - total_tokens_per_request=total_tokens_per_request, - # Latency metrics (convert to consistent units) - request_latency_median=request_latency_median / 1000.0, # Convert ms to seconds - request_latency_p95=request_latency_p95 / 1000.0, # Convert ms to seconds - ttft_median=ttft_median / 1000.0, # Convert ms to seconds - ttft_p95=ttft_p95 / 1000.0, # Convert ms to seconds - itl_median=itl_median / 1000.0, # Convert ms to seconds - itl_p95=itl_p95 / 1000.0, # Convert ms to seconds - tpot_median=tpot_median / 1000.0, # Convert ms to seconds - tpot_p95=tpot_p95 / 1000.0, # Convert ms to seconds - # Throughput metrics - tokens_per_second=total_tokens_per_second, - input_tokens_per_second=input_tokens_per_second, - output_tokens_per_second=output_tokens_per_second, - ) - - benchmarks.append(benchmark) - logging.info( - f"Parsed Guidellm benchmark: {strategy}, rate={request_rate:.2f} req/s, concurrency={concurrency_mean:.1f}, tokens/s={total_tokens_per_second:.1f}" - ) - - # Highlight unexpected throughput values - if total_tokens_per_second > 1000 or total_tokens_per_second < 1: - logging.warning( - f"Unusual tokens/second value: {total_tokens_per_second:.1f} for strategy {strategy}" - ) - - # Debug logging for troubleshooting - logging.debug(f" Raw parsed values for {strategy}:") - logging.debug( - f" Latency: TTFT={ttft_median:.3f}ms, TPOT={tpot_median:.3f}ms, ITL={itl_median:.3f}ms" - ) - logging.debug( - f" Request Latency: median={request_latency_median:.3f}ms, p95={request_latency_p95:.3f}ms" - ) - logging.debug( - f" Throughput: input={input_tokens_per_second:.1f}, output={output_tokens_per_second:.1f}, total={total_tokens_per_second:.1f}" - ) - logging.debug(f" Raw table rows:") - logging.debug(f" Summary: {summary_row}") - logging.debug(f" Latency: {latency_rows[i]}") - logging.debug(f" Throughput: {throughput_rows[i]}") - - except (ValueError, IndexError) as e: - logging.warning(f"Failed to parse Guidellm benchmark row {i}: {e}") - logging.debug(f" Problem row data:") - logging.debug(f" Summary row: {summary_row}") - if i < len(latency_rows): - logging.debug(f" Latency row: {latency_rows[i]}") - if i < len(throughput_rows): - logging.debug(f" Throughput row: {throughput_rows[i]}") - continue - - return benchmarks - - -def parse_guidellm_benchmark_json(dirname, json_file_path: pathlib.Path) -> list[GuidellmBenchmark]: - """Parse Guidellm benchmark JSON file and extract metrics""" - - if not (dirname / json_file_path).exists(): - logging.warning(f"Guidellm benchmark JSON not found: {json_file_path}") - return [] - - try: - with open(register_important_file(dirname, json_file_path)) as f: - json_data = json.load(f) - - logging.debug( - f"Parsing GuideLLM JSON file: {json_file_path} (found {len(json_data.get('benchmarks', []))} benchmarks)" - ) - - benchmarks = [] - - # Parse each benchmark in the JSON - for benchmark_data in json_data.get("benchmarks", []): - try: - # Extract strategy and concurrency info with fallback logic - scheduler = benchmark_data.get("scheduler", {}) - config = benchmark_data.get("config", {}) - strategy_info = config.get("strategy", {}) - strategy = strategy_info.get("type_", "unknown") - - # Try multiple paths for concurrency extraction - concurrency = 1.0 - concurrency_source = "default" - try: - # First try: scheduler.strategy.streams - concurrency = float(strategy_info.get("streams", 0)) - if concurrency <= 0: - raise ValueError("Invalid streams value") - concurrency_source = "config.strategy.streams" - except (ValueError, TypeError): - try: - # Second try: profile.stream - sched_strategy = scheduler.get("strategy", {}) - streams = sched_strategy.get("streams") - if not streams or streams <= 0: - raise ValueError("No valid streams found") - concurrency = float(streams) - concurrency_source = "profile.scheduler.streams" - - except (ValueError, TypeError, IndexError): - logging.warning( - f"Could not find concurrency 'streams' for benchmark. Using default value 1.0" - ) - concurrency = 1.0 - concurrency_source = "default" - - # Extract timing info - state = scheduler.get("state", {}) - start_time = state.get("start_time", 0) - end_time = state.get("end_time", 0) - duration = end_time - start_time if end_time > start_time else 60.0 - - # Extract metrics - metrics = benchmark_data.get("metrics", {}) - - # Helper function to safely extract metric values - def get_metric_value( - metric_name, stat_type="median", default=0.0, fail_if_missing=False - ): - metric_data = metrics.get(metric_name, {}).get("successful", {}) - if stat_type in ["p95", "p90", "p75", "p50", "p25", "p10"]: - percentiles = metric_data.get("percentiles", {}) - if stat_type not in percentiles: - if fail_if_missing: - raise KeyError( - f"Percentile {stat_type} not found in {metric_name} percentiles: {list(percentiles.keys())}" - ) - return default - return float(percentiles[stat_type]) - else: - if stat_type not in metric_data: - if fail_if_missing: - raise KeyError( - f"Stat type {stat_type} not found in {metric_name} successful data: {list(metric_data.keys())}" - ) - return default - return float(metric_data[stat_type]) - - # Extract latency metrics (convert ms to seconds for consistency) - request_latency_median = get_metric_value("request_latency", "median") / 1000.0 - request_latency_p95 = get_metric_value("request_latency", "p95") / 1000.0 - - # Extract TTFT percentiles - ttft_median = get_metric_value("time_to_first_token_ms", "median") / 1000.0 - ttft_p10 = get_metric_value("time_to_first_token_ms", "p10") / 1000.0 - ttft_p25 = get_metric_value("time_to_first_token_ms", "p25") / 1000.0 - ttft_p50 = get_metric_value("time_to_first_token_ms", "p50") / 1000.0 - ttft_p75 = get_metric_value("time_to_first_token_ms", "p75") / 1000.0 - ttft_p90 = get_metric_value("time_to_first_token_ms", "p90") / 1000.0 - ttft_p95 = get_metric_value("time_to_first_token_ms", "p95") / 1000.0 - - # Extract ITL percentiles - itl_median = get_metric_value("inter_token_latency_ms", "median") / 1000.0 - itl_p10 = get_metric_value("inter_token_latency_ms", "p10") / 1000.0 - itl_p25 = get_metric_value("inter_token_latency_ms", "p25") / 1000.0 - itl_p50 = get_metric_value("inter_token_latency_ms", "p50") / 1000.0 - itl_p75 = get_metric_value("inter_token_latency_ms", "p75") / 1000.0 - itl_p90 = get_metric_value("inter_token_latency_ms", "p90") / 1000.0 - itl_p95 = get_metric_value("inter_token_latency_ms", "p95") / 1000.0 - - # Extract TPOT percentiles - tpot_median = get_metric_value("time_per_output_token_ms", "median") / 1000.0 - tpot_p95 = get_metric_value("time_per_output_token_ms", "p95") / 1000.0 - - # Extract throughput metrics - request_rate = get_metric_value("requests_per_second", "mean") - input_tokens_per_second = get_metric_value("input_tokens_per_second", "mean") - output_tokens_per_second = get_metric_value("output_tokens_per_second", "mean") - total_tokens_per_second = input_tokens_per_second + output_tokens_per_second - - # Extract output token percentiles (will fail if not available) - output_tokens_per_second_p10 = get_metric_value( - "output_tokens_per_second", "p10", fail_if_missing=True - ) - output_tokens_per_second_p25 = get_metric_value( - "output_tokens_per_second", "p25", fail_if_missing=True - ) - output_tokens_per_second_p50 = get_metric_value( - "output_tokens_per_second", "p50", fail_if_missing=True - ) - output_tokens_per_second_p75 = get_metric_value( - "output_tokens_per_second", "p75", fail_if_missing=True - ) - output_tokens_per_second_p90 = get_metric_value( - "output_tokens_per_second", "p90", fail_if_missing=True - ) - - # Calculate requests completed and tokens per request - completed_requests = int(request_rate * duration) if request_rate > 0 else 0 - input_tokens_per_request = ( - (input_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - output_tokens_per_request = ( - (output_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - total_tokens_per_request = ( - (total_tokens_per_second / request_rate) if request_rate > 0 else 0.0 - ) - - # Create GuidellmBenchmark object - benchmark = GuidellmBenchmark( - strategy=strategy, - duration=duration, - warmup_time=0.0, # Not available in JSON format - cooldown_time=0.0, # Not available in JSON format - # Request metrics - request_rate=request_rate, - request_concurrency=concurrency, - completed_requests=completed_requests, - failed_requests=0, # Could extract from unsuccessful metrics if needed - # Token metrics per request - input_tokens_per_request=input_tokens_per_request, - output_tokens_per_request=output_tokens_per_request, - total_tokens_per_request=total_tokens_per_request, - # Latency metrics (already in seconds) - request_latency_median=request_latency_median, - request_latency_p95=request_latency_p95, - ttft_median=ttft_median, - ttft_p10=ttft_p10, - ttft_p25=ttft_p25, - ttft_p50=ttft_p50, - ttft_p75=ttft_p75, - ttft_p90=ttft_p90, - ttft_p95=ttft_p95, - itl_median=itl_median, - itl_p10=itl_p10, - itl_p25=itl_p25, - itl_p50=itl_p50, - itl_p75=itl_p75, - itl_p90=itl_p90, - itl_p95=itl_p95, - tpot_median=tpot_median, - tpot_p95=tpot_p95, - # Throughput metrics - tokens_per_second=total_tokens_per_second, - input_tokens_per_second=input_tokens_per_second, - output_tokens_per_second=output_tokens_per_second, - # Output token percentiles - output_tokens_per_second_p10=output_tokens_per_second_p10, - output_tokens_per_second_p25=output_tokens_per_second_p25, - output_tokens_per_second_p50=output_tokens_per_second_p50, - output_tokens_per_second_p75=output_tokens_per_second_p75, - output_tokens_per_second_p90=output_tokens_per_second_p90, - ) - - benchmarks.append(benchmark) - logging.info( - f"Parsed JSON benchmark: {strategy}, rate={request_rate:.2f} req/s, concurrency={concurrency:.1f} (from {concurrency_source}), tokens/s={total_tokens_per_second:.1f}" - ) - - # Debug logging - logging.debug(f" JSON parsed values for {strategy}:") - logging.debug( - f" Concurrency: {concurrency:.1f} (extracted from {concurrency_source})" - ) - logging.debug( - f" Latency: TTFT={ttft_median * 1000:.3f}ms, TPOT={tpot_median * 1000:.3f}ms, ITL={itl_median * 1000:.3f}ms" - ) - logging.debug( - f" Request Latency: median={request_latency_median * 1000:.3f}ms, p95={request_latency_p95 * 1000:.3f}ms" - ) - logging.debug( - f" Throughput: input={input_tokens_per_second:.1f}, output={output_tokens_per_second:.1f}, total={total_tokens_per_second:.1f}" - ) - - except (KeyError, ValueError, TypeError) as e: - logging.warning(f"Failed to parse benchmark data in JSON: {e}") - logging.debug(f"Problematic benchmark data: {benchmark_data}") - continue - - return benchmarks - - except (json.JSONDecodeError, ValueError) as e: - logging.warning(f"Failed to parse Guidellm JSON {json_file_path}: {e}") - return [] - - -def _parse_guidellm_config(dirname, json_file_path: pathlib.Path): - """Parse GuideLLM configuration from JSON file""" - - if not (dirname / json_file_path).exists(): - logging.warning(f"GuideLLM JSON config file not found: {json_file_path}") - return None - - try: - with open(register_important_file(dirname, json_file_path)) as f: - json_data = json.load(f) - - # Extract top-level configuration fields - config = {} - - # Extract args if present - if "args" in json_data and json_data["args"]: - config["args"] = json_data["args"] - if "metadata" in json_data and json_data["metadata"]: - config["metadata"] = json_data["metadata"] - - logging.info(f"Successfully parsed GuideLLM configuration from {json_file_path}") - - return config if config else None - - except Exception as e: - logging.error(f"Failed to parse GuideLLM configuration from {json_file_path}: {e}") - return None - - -def _parse_llmisvc_config(dirname): - """Parse LLMISVC configuration from captured state artifacts""" - if artifact_paths.LLMISVC_CAPTURE_DIR is None: - logging.warning("LLMISVC capture directory not found") - return None - - llmisvc_config_path = ( - artifact_paths.LLMISVC_CAPTURE_DIR / "artifacts" / "llminferenceservice.json" - ) - - if not (dirname / llmisvc_config_path).exists(): - logging.warning(f"LLMISVC config file not found at {llmisvc_config_path}") - return None - - try: - with open(register_important_file(dirname, llmisvc_config_path)) as f: - llmisvc_config = json.load(f) - logging.info(f"Successfully parsed LLMISVC configuration from {llmisvc_config_path}") - return llmisvc_config - except Exception as e: - logging.error(f"Failed to parse LLMISVC configuration from {llmisvc_config_path}: {e}") - return None - - -def _extract_metrics(dirname): - """Extract Prometheus metrics from main and UWM database files""" - db_files = {} - - # Add main Prometheus metrics (cluster-level: node, kube, nvidia, etc.) - if artifact_paths.PROMETHEUS_DUMP_DIR is not None: - db_files["sutest"] = ( - str(artifact_paths.PROMETHEUS_DUMP_DIR / "prometheus.t*"), - get_llmd_main_metrics(), - ) - - # Add UWM Prometheus metrics (application-level: vllm, etc.) - if artifact_paths.PROMETHEUS_UWM_DUMP_DIR is not None: - db_files["uwm"] = ( - str(artifact_paths.PROMETHEUS_UWM_DUMP_DIR / "prometheus.t*"), - get_llmd_uwm_metrics(), - ) - - if not db_files: - logging.info("No Prometheus DB directories found, skipping metrics extraction") - return None - - return helpers_store_parsers.extract_metrics(dirname, db_files) - - -SUTEST_CONTAINER_LABELS = [ - {"LLM Inference Service": dict(namespace="llm-d-project", pod=".*-kserve-.*")}, - {"LLM Inference Gateway": dict(namespace="llm-d-project", pod=".*-epp-.*")}, -] - -SUTEST_CONTAINER_EXTRA_METRICS_NAMES = [ - "LLM Inference Service", - "LLM Inference Gateway", -] - - -def get_llmd_main_metrics(register=False): - """Define the list of main Prometheus metrics to extract for LLM-D inference workloads""" - - cluster_role = "sutest" - - all_metrics = [] - - all_metrics += [{"up": "up"}] # for the test start/end timestamp - - all_metrics += helper_prom_store.get_cluster_metrics( - cluster_role, - register=register, - container_labels=SUTEST_CONTAINER_LABELS, - gpu_container="main", - network_metrics_names=SUTEST_CONTAINER_EXTRA_METRICS_NAMES, - ) - - return all_metrics - - -def get_llmd_uwm_metrics(): - """Define the list of UWM Prometheus metrics to extract for LLM-D inference workloads""" - - # Base metrics - always include 'up' for timestamp reference - all_metrics = [] - all_metrics += [{"up": "up"}] # for the test start/end timestamp - - # ============================================================================= - # 🔥 APPLICATION-LEVEL VLLM METRICS (stored in UWM Prometheus) - # ============================================================================= - - # End-to-end request latency (most critical for user experience) - all_metrics += [ - {"vllm_e2e_latency_seconds_bucket": "kserve_vllm:e2e_request_latency_seconds_bucket"}, - {"vllm_e2e_latency_seconds_sum": "kserve_vllm:e2e_request_latency_seconds_sum"}, - {"vllm_e2e_latency_seconds_count": "kserve_vllm:e2e_request_latency_seconds_count"}, - ] - - # Time to First Token (TTFT) - critical for streaming response feel - all_metrics += [ - {"vllm_ttft_seconds_bucket": "kserve_vllm:time_to_first_token_seconds_bucket"}, - {"vllm_ttft_seconds_sum": "kserve_vllm:time_to_first_token_seconds_sum"}, - {"vllm_ttft_seconds_count": "kserve_vllm:time_to_first_token_seconds_count"}, - ] - - # Inter-token latency - streaming quality - all_metrics += [ - { - "vllm_inter_token_latency_seconds_bucket": "kserve_vllm:inter_token_latency_seconds_bucket" - }, - {"vllm_inter_token_latency_seconds_sum": "kserve_vllm:inter_token_latency_seconds_sum"}, - {"vllm_inter_token_latency_seconds_count": "kserve_vllm:inter_token_latency_seconds_count"}, - ] - - # Request processing phases - all_metrics += [ - { - "vllm_request_prefill_time_seconds_bucket": "kserve_vllm:request_prefill_time_seconds_bucket" - }, - {"vllm_request_prefill_time_seconds_sum": "kserve_vllm:request_prefill_time_seconds_sum"}, - { - "vllm_request_prefill_time_seconds_count": "kserve_vllm:request_prefill_time_seconds_count" - }, - { - "vllm_request_decode_time_seconds_bucket": "kserve_vllm:request_decode_time_seconds_bucket" - }, - {"vllm_request_decode_time_seconds_sum": "kserve_vllm:request_decode_time_seconds_sum"}, - {"vllm_request_decode_time_seconds_count": "kserve_vllm:request_decode_time_seconds_count"}, - {"vllm_request_queue_time_seconds_bucket": "kserve_vllm:request_queue_time_seconds_bucket"}, - {"vllm_request_queue_time_seconds_sum": "kserve_vllm:request_queue_time_seconds_sum"}, - {"vllm_request_queue_time_seconds_count": "kserve_vllm:request_queue_time_seconds_count"}, - ] - - # Token throughput metrics - key performance indicators - all_metrics += [ - {"vllm_prompt_tokens_total": "kserve_vllm:prompt_tokens_total"}, - {"vllm_generation_tokens_total": "kserve_vllm:generation_tokens_total"}, - {"vllm_request_prompt_tokens_bucket": "kserve_vllm:request_prompt_tokens_bucket"}, - {"vllm_request_generation_tokens_bucket": "kserve_vllm:request_generation_tokens_bucket"}, - { - "vllm_request_max_num_generation_tokens_bucket": "kserve_vllm:request_max_num_generation_tokens_bucket" - }, - { - "vllm_request_max_num_generation_tokens_sum": "kserve_vllm:request_max_num_generation_tokens_sum" - }, - { - "vllm_request_max_num_generation_tokens_count": "kserve_vllm:request_max_num_generation_tokens_count" - }, - ] - - # Request queue state and capacity - all_metrics += [ - {"vllm_num_requests_running": "kserve_vllm:num_requests_running"}, - {"vllm_num_requests_waiting": "kserve_vllm:num_requests_waiting"}, - {"vllm_kv_cache_usage_perc": "kserve_vllm:kv_cache_usage_perc"}, - {"vllm_request_success_total": "kserve_vllm:request_success_total"}, - ] - - # ============================================================================= - # 🔧 COMPUTED VLLM METRICS (rate, average, increase transformations) - # ============================================================================= - - # E2E Request Latency computed metrics - all_metrics += [ - { - "vllm_e2e_latency_average": "rate(kserve_vllm:e2e_request_latency_seconds_sum[5m]) / rate(kserve_vllm:e2e_request_latency_seconds_count[5m])" - }, - {"vllm_e2e_request_rate": "rate(kserve_vllm:e2e_request_latency_seconds_count[5m])"}, - ] - - # TTFT computed metrics - all_metrics += [ - { - "vllm_ttft_average": "rate(kserve_vllm:time_to_first_token_seconds_sum[5m]) / rate(kserve_vllm:time_to_first_token_seconds_count[5m])" - }, - {"vllm_ttft_rate": "rate(kserve_vllm:time_to_first_token_seconds_count[5m])"}, - ] - - # Inter-token latency computed metrics - all_metrics += [ - { - "vllm_inter_token_average": "rate(kserve_vllm:inter_token_latency_seconds_sum[5m]) / rate(kserve_vllm:inter_token_latency_seconds_count[5m])" - }, - {"vllm_inter_token_rate": "rate(kserve_vllm:inter_token_latency_seconds_count[5m])"}, - ] - - # Request processing phase computed metrics - all_metrics += [ - { - "vllm_prefill_average": "rate(kserve_vllm:request_prefill_time_seconds_sum[5m]) / rate(kserve_vllm:request_prefill_time_seconds_count[5m])" - }, - {"vllm_prefill_rate": "rate(kserve_vllm:request_prefill_time_seconds_sum[5m])"}, - { - "vllm_decode_average": "rate(kserve_vllm:request_decode_time_seconds_sum[5m]) / rate(kserve_vllm:request_decode_time_seconds_count[5m])" - }, - {"vllm_decode_rate": "rate(kserve_vllm:request_decode_time_seconds_sum[5m])"}, - { - "vllm_queue_average": "rate(kserve_vllm:request_queue_time_seconds_sum[5m]) / rate(kserve_vllm:request_queue_time_seconds_count[5m])" - }, - {"vllm_queue_rate": "rate(kserve_vllm:request_queue_time_seconds_sum[5m])"}, - ] - - # Token throughput computed metrics - all_metrics += [ - {"vllm_prompt_tokens_rate": "rate(kserve_vllm:prompt_tokens_total[5m])"}, - {"vllm_generation_tokens_rate": "rate(kserve_vllm:generation_tokens_total[5m])"}, - { - "vllm_total_tokens_rate": "rate(kserve_vllm:prompt_tokens_total[5m]) + rate(kserve_vllm:generation_tokens_total[5m])" - }, - { - "vllm_avg_max_gen_tokens": "rate(kserve_vllm:request_max_num_generation_tokens_sum[5m]) / rate(kserve_vllm:request_max_num_generation_tokens_count[5m])" - }, - ] - - # Request success computed metrics - all_metrics += [ - {"vllm_request_success_rate": "rate(kserve_vllm:request_success_total[5m])"}, - {"vllm_request_success_increase": "increase(kserve_vllm:request_success_total[5m])"}, - ] - - return all_metrics From ce0c897983e20441544b57f5f49623d249f5c5f7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:18:59 +0200 Subject: [PATCH 0880/1072] [jump_ci] Remove --- projects/jump_ci/orchestration/ci.py | 138 -------- projects/jump_ci/readme.md | 11 - projects/jump_ci/testing/command_args.yml.j2 | 1 - projects/jump_ci/testing/config.yaml | 102 ------ projects/jump_ci/testing/prepare_jump_ci.py | 145 -------- projects/jump_ci/testing/run | 41 --- projects/jump_ci/testing/test.py | 330 ------------------ projects/jump_ci/testing/tunnelling.py | 223 ------------ projects/jump_ci/testing/utils.py | 102 ------ projects/jump_ci/testing/utils_gethostname.py | 118 ------- projects/jump_ci/toolbox/jump_ci.py | 169 --------- .../defaults/main/config.yml | 19 - .../toolbox/jump_ci_ensure_lock/files/.keep | 0 .../jump_ci_ensure_lock/tasks/main.yml | 46 --- .../jump_ci_ensure_lock/templates/.keep | 0 .../vars/main/resources.yml | 0 .../defaults/main/config.yml | 35 -- .../toolbox/jump_ci_prepare_step/files/.keep | 0 .../jump_ci_prepare_step/tasks/main.yml | 104 ------ .../jump_ci_prepare_step/templates/.keep | 0 .../templates/entrypoint.sh.j2 | 68 ---- .../vars/main/resources.yml | 1 - .../defaults/main/config.yml | 43 --- .../jump_ci_prepare_topsail/files/.keep | 0 .../jump_ci_prepare_topsail/tasks/main.yml | 196 ----------- .../jump_ci_prepare_topsail/templates/.keep | 0 .../vars/main/resources.yml | 0 .../defaults/main/config.yml | 16 - .../toolbox/jump_ci_release_lock/files/.keep | 0 .../jump_ci_release_lock/tasks/main.yml | 12 - .../jump_ci_release_lock/templates/.keep | 0 .../vars/main/resources.yml | 0 .../defaults/main/config.yml | 26 -- .../jump_ci_retrieve_artifacts/files/.keep | 0 .../jump_ci_retrieve_artifacts/tasks/main.yml | 24 -- .../templates/.keep | 0 .../vars/main/resources.yml | 0 .../defaults/main/config.yml | 16 - .../toolbox/jump_ci_take_lock/files/.keep | 0 .../toolbox/jump_ci_take_lock/tasks/main.yml | 35 -- .../toolbox/jump_ci_take_lock/templates/.keep | 0 .../jump_ci_take_lock/vars/main/resources.yml | 0 42 files changed, 2021 deletions(-) delete mode 100755 projects/jump_ci/orchestration/ci.py delete mode 100644 projects/jump_ci/readme.md delete mode 100644 projects/jump_ci/testing/command_args.yml.j2 delete mode 100644 projects/jump_ci/testing/config.yaml delete mode 100644 projects/jump_ci/testing/prepare_jump_ci.py delete mode 100755 projects/jump_ci/testing/run delete mode 100755 projects/jump_ci/testing/test.py delete mode 100644 projects/jump_ci/testing/tunnelling.py delete mode 100644 projects/jump_ci/testing/utils.py delete mode 100644 projects/jump_ci/testing/utils_gethostname.py delete mode 100644 projects/jump_ci/toolbox/jump_ci.py delete mode 100644 projects/jump_ci/toolbox/jump_ci_ensure_lock/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_ensure_lock/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_ensure_lock/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_ensure_lock/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_ensure_lock/vars/main/resources.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/templates/entrypoint.sh.j2 delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_step/vars/main/resources.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_topsail/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_topsail/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_topsail/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_topsail/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_prepare_topsail/vars/main/resources.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_release_lock/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_release_lock/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_release_lock/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_release_lock/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_release_lock/vars/main/resources.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/vars/main/resources.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_take_lock/defaults/main/config.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_take_lock/files/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_take_lock/tasks/main.yml delete mode 100644 projects/jump_ci/toolbox/jump_ci_take_lock/templates/.keep delete mode 100644 projects/jump_ci/toolbox/jump_ci_take_lock/vars/main/resources.yml diff --git a/projects/jump_ci/orchestration/ci.py b/projects/jump_ci/orchestration/ci.py deleted file mode 100755 index 0a6b8f886..000000000 --- a/projects/jump_ci/orchestration/ci.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -""" -Jump CI Project CI Operations - -This is the JumpCI CI entrypoint. It's used to run TOPSAIL-ng remotely inside a VPN cluster""" - -import sys -import traceback -from pathlib import Path - -import click - -from projects.core.library import ci as ci_lib - -# Add the testing directory to path for imports -testing_dir = Path(__file__).parent.parent / "testing" -if str(testing_dir) not in sys.path: - sys.path.insert(0, str(testing_dir)) - -# Import jump CI testing functionality -try: - import prepare_jump_ci as prepare_jump_ci_mod - from test import JumpCi -except ImportError as e: - raise RuntimeError(f"Jump CI testing functionality not available: {e}") from e - - -def log(message: str, level: str = "info"): - """Log message with project prefix.""" - project_name = "jump ci" - icon = {"info": "ℹ️", "success": "✅", "error": "❌", "warning": "⚠️"}.get(level, "ℹ️") - click.echo(f"{icon} [{project_name}] {message}") - - -def run_and_catch(phase, fct, *args, **kwargs): - try: - return fct(*args, **kwargs) - except Exception as e: - log(f"{phase} phase failed: {e}", "error") - log(f"Full stack-trace:\n{traceback.format_exc()}") - sys.exit(1) - - -@click.group(cls=ci_lib.HelpfulGroup) -@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") -@click.pass_context -def main(ctx, verbose): - """Jump CI Project CI Operations for TOPSAIL-NG.""" - ctx.ensure_object(dict) - ctx.obj["verbose"] = verbose - - -@main.command() -@click.pass_context -@click.option( - "--cluster", - "cluster", - nargs=1, - metavar="KEY", - help="Give the name of the cluster to lock", - default=None, -) -def lock_cluster(ctx, cluster): - """Prepare phase - Lock the cluster.""" - log("Locking cluster for exclusive access...") - try: - prepare_jump_ci_mod.lock_cluster(cluster) - log("Cluster locked successfully", "success") - sys.exit(0) - except Exception as e: - log(f"Lock phase failed: {e}", "error") - log(f"Full stack-trace:\n{traceback.format_exc()}") - - sys.exit(1) - - -@main.command() -@click.pass_context -def prepare_jump_ci(ctx): - """Prepare phase - Prepare the Jump CI remote system.""" - - log("Starting prepare_jump_ci phase...") - run_and_catch("prepare_jump_ci", prepare_jump_ci_mod.prepare) - log("Jump CI environment prepared", "success") - sys.exit(0) - - -@main.command() -@click.pass_context -@click.option( - "--cluster", - "cluster", - nargs=1, - metavar="KEY", - help="Give the name of the cluster to unlock", - default=None, -) -def unlock_cluster(ctx, cluster): - """Teardown phase - Unlock the cluster.""" - log("Unlocking the cluster...") - run_and_catch("unlock_cluster", prepare_jump_ci_mod.unlock_cluster, cluster) - log("Cluster unlocked successfully", "success") - sys.exit(0) - - -@main.command() -@click.pass_context -def prepare(ctx): - """Prepare phase - Trigger the project's prepare method.""" - log("Starting prepare phase...") - ret = run_and_catch("prepare", JumpCi().prepare) - log("Project prepared", "success" if not ret else "error") - sys.exit(ret) - - -@main.command() -@click.pass_context -def test(ctx): - """Test phase - Trigger the project's test method.""" - log("Starting test phase...") - - ret = run_and_catch("test", JumpCi().test) - log("Jump CI environment test completed", "success" if not ret else "error") - sys.exit(0) - - -@main.command() -@click.pass_context -def pre_cleanup(ctx): - """Cleanup phase - Pre-clean up resources.""" - log("Starting cleanup phase...") - ret = run_and_catch("pre_cleanup", JumpCi().pre_cleanup) - log("Cleanup phase completed!", "success" if not ret else "error") - sys.exit(ret) - - -if __name__ == "__main__": - main() diff --git a/projects/jump_ci/readme.md b/projects/jump_ci/readme.md deleted file mode 100644 index 189980141..000000000 --- a/projects/jump_ci/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -To build the Jump-CI `main` image in the jump host: - -``` -ssh $JUMP_HOST -git clone https://github.com/openshift-psap/topsail.git -git submodule update --init - -cd topsail - -podman build -f ./build/Dockerfile --label preserve=true -t localhost/topsail:main . -``` diff --git a/projects/jump_ci/testing/command_args.yml.j2 b/projects/jump_ci/testing/command_args.yml.j2 deleted file mode 100644 index e16d8b253..000000000 --- a/projects/jump_ci/testing/command_args.yml.j2 +++ /dev/null @@ -1 +0,0 @@ -# not used diff --git a/projects/jump_ci/testing/config.yaml b/projects/jump_ci/testing/config.yaml deleted file mode 100644 index d2964835f..000000000 --- a/projects/jump_ci/testing/config.yaml +++ /dev/null @@ -1,102 +0,0 @@ -ci_presets: - # list of names of presets that have been applied - names: null - - mac_ai: - secrets.dir.name: crc-mac-ai-secret # key/value never actually used - secrets.dir.env_key: CRC_MAC_AI_SECRET_PATH - cluster.name: mac - project.name: mac_ai - - mac_ai_mac5: - extends: [mac_ai] - cluster.name: mac5 - - mac_ai_nightly: - extends: [mac_ai_mac5] - project.args: [cpt_nightly] - - mac_ai_nightly_lightspeed: - extends: [mac_ai_mac5] - project.args: [cpt_nightly_lightspeed] - - cont_bench: - secrets.dir.name: cont-bench-secret # key/value never actually used - secrets.dir.env_key: CONTAINER_BENCH_SECRET_PATH - cluster.name: mac.jrodak - project.name: container_bench - - llama_cpp_presubmit: - secrets.dir.name: crc-mac-ai-secret # key/value never actually used - secrets.dir.env_key: CRC_MAC_AI_SECRET_PATH - cluster.name: mac5 - project.name: mac_ai - - plot_cluster: - cluster.name: plot - -secrets: - dir: - name: psap-ods-secret - env_key: PSAP_FORGE_JUMP_CI_SECRET_PATH - private_key_filename: jumpci_privatekey - bastion_host_filename: jumpci_bastion_host - bastion_host_user_filename: jumpci_bastion_host_user - jump_host_filename: jumpci_jump_host - -prepare: - job_name_to_preset: - pull-ci-openshift-psap-topsail-main-cont_bench-jump-ci: cont_bench - pull-ci-openshift-psap-topsail-main-mac_ai-jump-ci: mac_ai - pull-ci-openshift-psap-topsail-main-mac_ai-jump-ci-mac5: mac_ai_mac5 - periodic-ci-openshift-psap-topsail-main-mac_ai-jump-ci-cpt: mac_ai_nightly - periodic-ci-openshift-psap-topsail-main-mac_ai-jump-ci-cpt-lightspeed: mac_ai_nightly_lightspeed - pull-ci-openshift-psap-topsail-main-jump-ci-plot: plot_cluster - pull-ci-crc-org-llama.cpp-main-topsail: llama_cpp_presubmit -ssh_tunnel: - # creates a tunnel to the bastion via the jump host, if enabled - enabled: false - local_host_port: 2500 - verbose: false # will leak host names if enabled -cluster: - name: null -project: - name: null - args: null - -foreign_testing: # used only when testing from another repo that FORGE itself - forge: - repo: - owner: openshift-psap - name: forge - branch: main -ssh: - flags: - - -o StrictHostKeyChecking=no - - -o UserKnownHostsFile=/dev/null - - -o LogLevel=ERROR - - -o ServerAliveInterval=60 - - -o ServerAliveCountMax=3 -env: - pass_lists: - OPENSHIFT_CI_job: [JOB_TYPE, ENTRYPOINT_OPTIONS, JOB_NAME, JOB_SPEC, OPENSHIFT_CI, JOB_NAME_SAFE, BUILD_ID] - OPENSHIFT_CI_git_pr: [PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_REF, REPO_NAME, REPO_OWNER, PULL_BASE_SHA, JOB_NAME, PULL_TITLE, PULL_REFS, PULL_HEAD_REF] - OPENSHIFT_CI_forge: [FORGE_OPENSHIFT_CI_STEP_DIR, OPENSHIFT_CI_FORGE_FOREIGN_TESTING] - -rewrite_variables_overrides: - cluster_found_in_pr_args: false - -exec_list: - _only_: false - # keep these steps enabled for the jump-ci to work - lock_cluster: true - prepare_jump_ci: true - unlock_cluster: true - - pre_cleanup: null - prepare: null - test: null - post_cleanup: null -multi_run: - args: [] - stop_on_error: false diff --git a/projects/jump_ci/testing/prepare_jump_ci.py b/projects/jump_ci/testing/prepare_jump_ci.py deleted file mode 100644 index 4100de4a6..000000000 --- a/projects/jump_ci/testing/prepare_jump_ci.py +++ /dev/null @@ -1,145 +0,0 @@ -import json -import logging -import os - -from projects.jump_ci.testing import tunnelling, utils -from projects.legacy.library import config, run - - -@utils.entrypoint() -def lock_cluster(cluster=None): - """ - Take the lock on a given cluster - - Args: - cluster: Name of the cluster lock to use - """ - if os.environ.get("OPENSHIFT_CI") == "true": - config.project.set_config("ssh_tunnel.enabled", "true") - - if config.project.get_config("cluster.name", print=False) is None: - override_cluster = config.project.get_config("overrides.PR_POSITIONAL_ARG_1", None) - if not override_cluster: - raise ValueError( - "Expected to find the cluster name in overrides.PR_POSITIONAL_ARG_1 (1st test argument) or cluster.name, but none of them were set ..." - ) - - config.project.set_config("cluster.name", override_cluster) - config.project.set_config("rewrite_variables_overrides.cluster_found_in_pr_args", True) - - if cluster is None: - cluster = config.project.get_config("cluster.name") - - # Open the tunnel - tunnelling.prepare() - - run.run_toolbox("jump_ci", "take_lock", cluster=cluster, owner=utils.get_lock_owner()) - - tunnelling.run_with_ansible_ssh_conf(f"podman container rm --force 'topsail-on-{cluster}'") - - -@utils.entrypoint() -def unlock_cluster(cluster=None): - """ - Release the lock on a given cluster - - Args: - cluster: Name of the cluster lock to use - """ - # Open the tunnel - tunnelling.prepare() - - if cluster is None: - cluster = config.project.get_config("cluster.name") - - tunnelling.run_with_ansible_ssh_conf(f"podman container rm --force 'topsail-on-{cluster}'") - - run.run_toolbox("jump_ci", "release_lock", cluster=cluster, owner=utils.get_lock_owner()) - - -@utils.entrypoint() -def prepare( - cluster=None, - repo_owner="openshift-psap", - repo_name="topsail-ng", - git_ref=None, - pr_number=None, -): - """ - Prepares the jump-host for running TOPSAIL commands. - Args: - cluster: Name of the cluster to use - owner: Name of the Github repo owner - repo: Name of the TOPSAIL github repo - git_ref: Commit to use in the TOPSAIL repo - pr_number: PR number to use for the test. If none, use the main branch. - """ - - # Open the tunnel - tunnelling.prepare() - - if cluster is None: - cluster = config.project.get_config("cluster.name") - - # Lock the cluster - run.run_toolbox("jump_ci", "ensure_lock", cluster=cluster, owner=utils.get_lock_owner()) - - # Clone the Git Repository - # Build the image - prepare_topsail_args = { - "cluster": cluster, - "lock_owner": utils.get_lock_owner(), - } - - if os.environ.get("OPENSHIFT_CI") == "true": - if os.environ.get("OPENSHIFT_CI_TOPSAIL_FOREIGN_TESTING"): - logging.info( - "Not running from TOPSAIL repository. Using TOPSAIL foreign configuration." - ) - repo_owner = config.project.get_config("foreign_testing.topsail.repo.owner") - repo_name = config.project.get_config("foreign_testing.topsail.repo.name") - git_ref = config.project.get_config("foreign_testing.topsail.repo.branch") - - elif os.environ["JOB_NAME"].startswith("periodic"): - # periodic jobs don't have these env vars ... - job_spec = json.loads(os.environ["JOB_SPEC"]) - repo_owner = os.environ["REPO_OWNER"] = job_spec["extra_refs"][0]["org"] - repo_name = os.environ["REPO_NAME"] = job_spec["extra_refs"][0]["repo"] - git_ref = os.environ["PULL_PULL_SHA"] = job_spec["extra_refs"][0]["base_ref"] - else: - repo_owner = os.environ["REPO_OWNER"] - repo_name = os.environ["REPO_NAME"] - git_ref = os.environ["PULL_PULL_SHA"] - - prepare_topsail_args |= { - "repo_owner": repo_owner, - "repo_name": repo_name, - "git_ref": git_ref, - "pr_number": os.environ.get("PULL_NUMBER"), - } - elif any([pr_number, git_ref]): - if not all([repo_owner, repo_name, pr_number]): - logging.fatal( - "Missing parameters in the CLI arguments. Please pass at least --pr-number" - ) - raise SystemExit(1) - - prepare_topsail_args |= { - "repo_owner": repo_owner, - "repo_name": repo_name, - "pr_number": pr_number, - "git_ref": git_ref, - } - - else: - logging.fatal("No flag provided and couldn't determine the CI environment ... Aborting.") - logging.info("Outside of the CI, please pass at least --pr-number") - raise SystemExit(1) - - run.run_toolbox( - "jump_ci", - "prepare_topsail", - **prepare_topsail_args, - ) - - return None diff --git a/projects/jump_ci/testing/run b/projects/jump_ci/testing/run deleted file mode 100755 index 21290ae35..000000000 --- a/projects/jump_ci/testing/run +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -import logging -import subprocess -import sys - -import fire - -from projects.jump_ci.testing import prepare_jump_ci, tunnelling -from projects.legacy.library import configure_logging - -logging.getLogger().setLevel(logging.INFO) -configure_logging() - - -class JumpCi: - """ - Commands for launching the Jump CI - """ - - def __init__(self): - self.lock_cluster = prepare_jump_ci.lock_cluster - self.unlock_cluster = prepare_jump_ci.unlock_cluster - - self.test_tunnel = tunnelling.open_tunnel - self.prepare_jump_ci = prepare_jump_ci.prepare - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - fire.Fire(JumpCi()) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - logging.error(f"Command '{e.cmd}' failed --> {e.returncode}") - sys.exit(1) diff --git a/projects/jump_ci/testing/test.py b/projects/jump_ci/testing/test.py deleted file mode 100755 index 6852d5f74..000000000 --- a/projects/jump_ci/testing/test.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python - -import logging -import os -import pathlib -import shlex -import subprocess -import sys - -import fire - -from projects.jump_ci.testing import tunnelling, utils -from projects.legacy.library import config, configure_logging, env, run - -logging.getLogger().setLevel(logging.INFO) - -configure_logging() - -LOCK_DIR_PREFIX = "/tmp/topsail" - - -def test_jump_ci_skip_list(current_subcommand): - """ - Check if the current jump_ci subcommand should be skipped based on exec_list configuration - - Args: - current_subcommand: The jump_ci command to check (e.g., "prepare", "test", "pre_cleanup") - """ - logging.info(f"Currently running the jump_ci subcommand '{current_subcommand}'") - - exec_list = config.project.get_config("exec_list", None, print=False) - if exec_list is None: - logging.warning("The exec_list isn't defined in this project.") - exec_list = {} - - NOT_FOUND = object() - exec_this_subcommand = exec_list.get(current_subcommand, NOT_FOUND) - if exec_this_subcommand is NOT_FOUND: - logging.info( - f"Subcommand '{current_subcommand}' is not defined in the exec list. Executing this command by default." - ) - return - - if exec_this_subcommand is False: - logging.fatal( - f"Subcommand '{current_subcommand}' is disabled in the exec list. Stopping happily this execution." - ) - with open(env.ARTIFACT_DIR / "SKIPPED", "w") as f: - print("Skipped because part of the \\skip list", file=f) - raise SystemExit(0) - - if exec_this_subcommand is not True and exec_list.get("_only_", False): - logging.fatal( - f"Only flag is set, and subcommand '{current_subcommand}' is not enabled in the exec list. Stopping happily this execution." - ) - with open(env.ARTIFACT_DIR / "SKIPPED", "w") as f: - print("Skipped because not part of the \\only list", file=f) - raise SystemExit(0) - - # not in the skip list - # not the only command to execute - # continue happilly =:-) - - -def rewrite_variables_overrides(variable_overrides_dict, nb_args_to_eat): - new_variable_overrides = {} - - old_args = (variable_overrides_dict["PR_POSITIONAL_ARGS"] or "").split() - # remove the first args of the jump-ci test (project and cluster) - new_args = old_args[nb_args_to_eat:] - - new_args_str = new_variable_overrides["PR_POSITIONAL_ARGS"] = " ".join(new_args) - new_variable_overrides["PR_POSITIONAL_ARGS"] = variable_overrides_dict["PR_POSITIONAL_ARG_0"] - new_variable_overrides["PR_POSITIONAL_ARG_0"] = variable_overrides_dict["PR_POSITIONAL_ARG_0"] - - logging.info(f"New args to execute on the jump host: {new_args_str}") - - idx = 0 - for idx, value in enumerate(new_args): - new_variable_overrides[f"PR_POSITIONAL_ARG_{idx + 1}"] = value - next_pr_positional_arg_count = idx + nb_args_to_eat - - for k, v in variable_overrides_dict.items(): - if k.startswith("PR_POSITIONAL_ARG"): - continue - - NOT_FOUND = object() - found = config.project.get_config(k, NOT_FOUND, print=False, warn=False) - if found != NOT_FOUND: - # the key 'k' was found in the jump-ci config. - # It belongs to us, do *not* pass it over to the new test. - logging.info(f"NOT passing '{k}: {v}' to the new variables overrides") - - continue - - # the key 'k' was not found in the jump-ci config - # It doesn't belong to us, pass it to the new test - new_variable_overrides[k] = v - logging.info(f"Passing '{k}: {v}' to the new variables overrides") - - return new_variable_overrides, next_pr_positional_arg_count - - -def jump_ci(command): - @utils.entrypoint() - def do_jump_ci(cluster=None, project=None, test_args=None): - """ - Runs a command in the Jump Host. - - Args: - cluster: the name of the cluster to run on - project: the name of the project to launch - test_args: the test args to pass to the test command - """ - - # Check skip/only list for this jump_ci command - test_jump_ci_skip_list(command) - - # Open the tunnel - tunnelling.prepare() - - if cluster is None: - cluster = config.project.get_config("cluster.name") - - secrets_path_env_key = config.project.get_config("secrets.dir.env_key") - - extra_env = { - "FORGE_JUMP_CI": "true", - "FORGE_JUMP_CI_INSIDE_JUMP_HOST": "true", - } - - def prepare_env_file(_extra_env): - env_fd_path, env_file = utils.get_tmp_fd() - - env_pass_lists = config.project.get_config("env.pass_lists", print=False) - - env_pass_list = set() - for _, pass_list in (env_pass_lists or {}).items(): - env_pass_list.update(pass_list) - - for k, v in (os.environ | _extra_env).items(): - if k not in (env_pass_list | _extra_env.keys()): - continue - - print(f"{k}={shlex.quote(v)}", file=env_file) - - env_file.flush() - - return env_fd_path, env_file - - variable_overrides_file = ( - pathlib.Path(os.environ.get("ARTIFACT_DIR")) - / "000__ci_metadata" - / "variable_overrides.yaml" - ) - - config_test_args = ( - config.project.get_config("project.args", None) if test_args is None else None - ) - - if test_args is None and not variable_overrides_file.exists() and not config_test_args: - logging.fatal( - f"File '{variable_overrides_file}' does not exist, neither --test_args nor project.args have been passed. Please specify one of them :/" - ) - raise SystemExit(1) - - if test_args is not None and not project: - logging.fatal("The --project flag must be specificed when --test_args is passed") - raise SystemExit(1) - - run.run_toolbox("jump_ci", "ensure_lock", cluster=cluster, owner=utils.get_lock_owner()) - - cluster_lock_dir = f"{LOCK_DIR_PREFIX}_{cluster}" - - if not project: - project = config.project.get_config("project.name") - - if test_args is not None or config_test_args is not None: - if test_args is None: - test_args = " ".join(config_test_args) - - variables_overrides_dict = { - "PR_POSITIONAL_ARGS": test_args, - "PR_POSITIONAL_ARG_0": "jump-ci", - } - idx = 0 - for idx, arg in enumerate(test_args.split()): - variables_overrides_dict[f"PR_POSITIONAL_ARG_{idx + 1}"] = arg - - config.project.set_config("overrides", variables_overrides_dict) - next_pr_positional_arg_count = idx + 2 - else: - if not os.environ.get("OPENSHIFT_CI") == "true": - logging.fatal( - "Not running in OpenShift CI. Don't know how to rewrite the variable_overrides_file. Aborting." - ) - raise SystemExit(1) - - nb_args_to_eat = 0 - if config.project.get_config("rewrite_variables_overrides.cluster_found_in_pr_args"): - nb_args_to_eat = 1 - - if not project: - project = config.project.get_config( - f"overrides.PR_POSITIONAL_ARG_{1 + nb_args_to_eat}", None - ) - if not project: - raise ValueError( - f"Expected to find the project name in project.name or overrides.PR_POSITIONAL_ARG_{1 + nb_args_to_eat}, but both of them were empty :/" - ) - nb_args_to_eat += 1 - - variables_overrides_dict, next_pr_positional_arg_count = rewrite_variables_overrides( - config.project.get_config("overrides"), - nb_args_to_eat, - ) - - for idx, multi_run_args in enumerate(config.project.get_config("multi_run.args") or [...]): - multi_run_args_dict = {} - multi_run_dirname = None - test_artifacts_dirname = "test-artifacts" - - if multi_run_args is not ...: - multi_run_args_lst = ( - multi_run_args if isinstance(multi_run_args, list) else [multi_run_args] - ) - multi_run_dirname = f"multi_run__{'_'.join(multi_run_args_lst)}" - - with open(env.ARTIFACT_DIR / "multi_run_args.list", "a+"): - print(f"{multi_run_dirname}: {multi_run_args}") - - for idx, multi_run_arg in enumerate(multi_run_args_lst): - variables_overrides_dict[ - f"PR_POSITIONAL_ARG_{next_pr_positional_arg_count + idx}" - ] = multi_run_arg - - with env.NextArtifactDir(multi_run_dirname) if multi_run_dirname else open("/dev/null"): - if multi_run_dirname: - test_artifacts_dirname = f"{env.ARTIFACT_DIR.name}/{test_artifacts_dirname}" - - if step_dir := os.environ.get("FORGE_OPENSHIFT_CI_STEP_DIR"): - # see "jump_ci retrieve_artifacts" below - extra_env["FORGE_OPENSHIFT_CI_STEP_DIR"] = ( - f"{step_dir}/{test_artifacts_dirname}" - ) - - env_fd_path, env_file = prepare_env_file(extra_env) - - run.run_toolbox( - "jump_ci", - "prepare_step", - cluster=cluster, - lock_owner=utils.get_lock_owner(), - project=project, - step=command, - env_file=env_fd_path, - variables_overrides_dict=(variables_overrides_dict | multi_run_args_dict), - secrets_path_env_key=secrets_path_env_key, - ) - env_file.close() - - try: - tunnelling.run_with_ansible_ssh_conf( - f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh" - ) - logging.info(f"Test step '{command}' on cluster '{cluster}' succeeded.") - failed = False - except subprocess.CalledProcessError: - logging.fatal(f"Test step '{command}' on cluster '{cluster}' FAILED.") - failed = True - except run.SignalInterrupt as e: - logging.error(f"Caught signal {e.sig}. Aborting.") - raise - finally: - # always run the cleanup to be sure that the container doesn't stay running - tunnelling.run_with_ansible_ssh_conf( - f"bash {cluster_lock_dir}/test/{command}/entrypoint.sh cleanup" - ) - - run.run_toolbox( - "jump_ci", - "retrieve_artifacts", - cluster=cluster, - lock_owner=utils.get_lock_owner(), - remote_dir=f"test/{command}/artifacts", - local_dir=f"../{pathlib.Path(test_artifacts_dirname).name}", # copy to the main artifact directory - mute_stdout=True, - mute_stderr=True, - ) - - if failed and config.project.get_config("multi_run.stop_on_error"): - break - - jump_ci_artifacts = env.ARTIFACT_DIR / "jump-ci-artifacts" - jump_ci_artifacts.mkdir(parents=True, exist_ok=True) - run.run(f"mv {env.ARTIFACT_DIR}/*__jump_ci_* {jump_ci_artifacts}/") - - if failed: - raise SystemExit(1) - - return do_jump_ci - - -class JumpCi: - """ - Commands for launching the Jump CI - """ - - def __init__(self): - self.pre_cleanup = jump_ci("pre_cleanup") - self.post_cleanup = jump_ci("post_cleanup") - self.prepare = jump_ci("prepare") - self.test = jump_ci("test") - - self.generate_plots_from_pr_args = jump_ci("generate_plots_from_pr_args") - - -def main(): - # Print help rather than opening a pager - fire.core.Display = lambda lines, out: print(*lines, file=out) - - fire.Fire(JumpCi()) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - logging.error(f"Command '{e.cmd}' failed --> {e.returncode}") - sys.exit(1) diff --git a/projects/jump_ci/testing/tunnelling.py b/projects/jump_ci/testing/tunnelling.py deleted file mode 100644 index 89071c524..000000000 --- a/projects/jump_ci/testing/tunnelling.py +++ /dev/null @@ -1,223 +0,0 @@ -import atexit -import logging -import os -import pathlib -import subprocess -import time - -import yaml - -from projects.jump_ci.testing import utils -from projects.legacy.library import config, run - -extra_vars_file = None - - -@utils.entrypoint() -def prepare( - verbose=None, -): - """ - Prepare the environment for running TOPSAIL in the jump host - - Args: - verbose: if enabled, displays additional info. *Will display SSH host names*. - """ - - LOCAL_HOST_PORT = config.project.get_config("ssh_tunnel.local_host_port") - SECRET_ENV_KEY = config.project.get_config("secrets.dir.env_key") - PRIVATE_KEY_FILENAME = config.project.get_config("secrets.private_key_filename") - BASTION_HOST_FILENAME = config.project.get_config("secrets.bastion_host_filename") - BASTION_HOST_USER_FILENAME = config.project.get_config("secrets.bastion_host_user_filename") - JUMP_HOST_FILENAME = config.project.get_config("secrets.jump_host_filename") - SSH_FLAGS = config.project.get_config("ssh.flags") - - private_key_path = pathlib.Path(os.environ[SECRET_ENV_KEY]) / PRIVATE_KEY_FILENAME - - if verbose is None: - verbose = config.project.get_config("ssh_tunnel.verbose") - - with open(pathlib.Path(os.environ[SECRET_ENV_KEY]) / BASTION_HOST_FILENAME) as f: - bastion_host = f.readline().strip() - - with open(pathlib.Path(os.environ[SECRET_ENV_KEY]) / BASTION_HOST_USER_FILENAME) as f: - bastion_user = f.readline().strip() - - if config.project.get_config("ssh_tunnel.enabled"): - logging.info( - "ssh_tunnel.enabled is set, creating a tunnel to the bastion via the jump host" - ) - # creates a tunnel to the jumphost to the bastion, on localhost:2500 - - open_tunnel( - secret_env_key=SECRET_ENV_KEY, - private_key_filename=PRIVATE_KEY_FILENAME, - bastion_host_filename=BASTION_HOST_FILENAME, - bastion_host_user_filename=BASTION_HOST_USER_FILENAME, - local_host_port=LOCAL_HOST_PORT, - jump_host_filename=JUMP_HOST_FILENAME, - verbose=verbose, - keep_open=False, - ssh_flags=SSH_FLAGS, - ) - - remote_hostname = "localhost" - remote_host_port = LOCAL_HOST_PORT - else: - logging.info("ssh_tunnel.enabled is disabled, connecting directly to the bastion") - remote_hostname = bastion_host - remote_host_port = 22 - # probe_ssh_endpoint( - # bastion_user, - # bastion_host, - # remote_host_port, - # private_key_path, - # SSH_FLAGS, - # verbose, - # ) - - os.environ["TOPSAIL_REMOTE_HOSTNAME"] = remote_hostname - - extra_vars_fd_path, extra_vars_file = utils.get_tmp_fd() - - os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS"] = extra_vars_fd_path - - extra_vars_yaml_content = f""" -ansible_port: {remote_host_port} -ansible_ssh_private_key_file: {private_key_path} -ansible_ssh_user: {bastion_user} -ansible_ssh_common_args: "{" ".join(SSH_FLAGS)}" -""" - print(extra_vars_yaml_content, file=extra_vars_file) - extra_vars_file.flush() - - -@utils.entrypoint() -def open_tunnel( - secret_env_key="PSAP_ODS_SECRET_PATH", - private_key_filename="jumpci_privatekey", - jump_host_filename="jumpci_jump_host", - local_host_port=2500, - bastion_host_filename="jumpci_bastion_host", - bastion_host_user_filename="jumpci_bastion_host_user", - bastion_host_port=22, - verbose=False, - keep_open=True, - ssh_flags=None, -): - """ - Tests the tunnel to access the jump host. - - Args: - secret_env_key: name of the env key pointing to the secret directory - private_key_filename: name of the secret file in the secret directory - jump_host_filename: ssh name of the jump host - local_host_port: port to open locally - bastion_host_filename: name of the secret file with the hostname of the bastion host - bastion_host_user_filename: name of the secret file with the username of the bastion host - bastion_host_port: port to redirect in the target host - verbose: if enabled, shows the tunnel command - keep_open: if disabled, closes the tunnel when the Python process exits - ssh_flags: extra flags to pass to SSH - """ - - if ssh_flags is None: - ssh_flags = [] - secret_dir = pathlib.Path(os.environ[secret_env_key]) - - private_key_path = secret_dir / private_key_filename - - with open(secret_dir / jump_host_filename) as f: - jump_host = f.readline().strip() - - with open(secret_dir / bastion_host_filename) as f: - bastion_host = f.readline().strip() - - with open(secret_dir / bastion_host_user_filename) as f: - bastion_user = f.readline().strip() - - # warning: this command doesn't fail ... - cmd = f"ssh {' '.join(ssh_flags)} \ - -i {private_key_path} {jump_host} \ - -L {local_host_port}:{bastion_host}:{bastion_host_port} \ - -N" - - if verbose: - logging.info(cmd) - - RETRIES = 1 - DELAY = 5 - recreate_count_down = RECREATED_TUNNEL_COUNT_DOWN = ( - 5 # recreate the SSH tunnel every 5 attempts - ) - - def create_tunnel(): - _proc = subprocess.Popen(cmd, shell=True) - if not keep_open: - atexit.register(_proc.kill) - - time.sleep(DELAY) - - return _proc - - proc = create_tunnel() - - logging.info("Waiting for the SSH connection to work ...") - for i in range(RETRIES): - try: - probe_ssh_endpoint( - bastion_user, - "localhost", - local_host_port, - private_key_path, - ssh_flags, - verbose, - ) - logging.info("SSH connection working!") - break - except subprocess.CalledProcessError: - pass - recreate_count_down -= 1 - - logging.info(f"Attempt {i + 1}/{RETRIES} failed ...") - if i == (RETRIES - 1): - raise Exception("SSH connection probe failed :/") - - if recreate_count_down == 0: - logging.info("Killing the non-working SSH tunnel ...") - proc.kill() - - if not keep_open: - atexit.unregister(proc.kill) - logging.info("Creating a new SSH tunnel ...") - proc = create_tunnel() - recreate_count_down = RECREATED_TUNNEL_COUNT_DOWN - - time.sleep(DELAY) - - return - - -def probe_ssh_endpoint(user, host, port, private_key_path, ssh_flags, verbose): - run.run( - f"ssh {' '.join(ssh_flags)} -i {private_key_path} {user + '@' if user else ''}{host} -p {port} true", - capture_stderr=True, - log_command=verbose, - ) - - -def run_with_ansible_ssh_conf(cmd): - with open(os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS"]) as f: - ansible_ssh_config = yaml.safe_load(f) - - ssh_flags = ansible_ssh_config["ansible_ssh_common_args"] - hostname = os.environ["TOPSAIL_REMOTE_HOSTNAME"] - port = ansible_ssh_config["ansible_port"] - user = ansible_ssh_config["ansible_ssh_user"] - private_key_path = ansible_ssh_config["ansible_ssh_private_key_file"] - - logging.info(f"Running on the jump host: {cmd}") - run.run( - f"ssh {ssh_flags} -t -i {private_key_path} {user}@{hostname} -p {port} -- {cmd}", - log_command=False, - ) diff --git a/projects/jump_ci/testing/utils.py b/projects/jump_ci/testing/utils.py deleted file mode 100644 index 79a16e7ee..000000000 --- a/projects/jump_ci/testing/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -import functools -import logging -import os -import pathlib -import tempfile - -import utils_gethostname - -from projects.legacy.library import config, env - -TESTING_THIS_DIR = pathlib.Path(__file__).absolute().parent - - -def apply_preset_by_jobname(): - if os.environ.get("OPENSHIFT_CI") != "true": - logging.info("apply_preset_by_jobname: not running in Openshift CI, skipping") - return - - job_name = os.environ["JOB_NAME"] - - presets = config.project.get_config("prepare.job_name_to_preset") - preset_name = presets.get(job_name) - if not preset_name: - logging.info(f"apply_preset_by_jobname: no preset to apply for '{job_name}'") - return - - logging.info( - f"apply_preset_by_jobname: applying the '{preset_name}' preset for job name '{job_name}' ..." - ) - config.project.apply_preset(preset_name) - - -initialized = False - - -def init(ignore_secret_path=False, apply_preset_from_pr_args=True): - global initialized - if initialized: - logging.debug("Already initialized.") - return - initialized = True - - env.init() - config.init(TESTING_THIS_DIR, apply_config_overrides=False) - config.project.apply_config_overrides(ignore_not_found=True) - apply_preset_by_jobname() - config.project.apply_config_overrides(ignore_not_found=True) - config.test_skip_list() - - node_name = utils_gethostname.get_kubernetes_node_name() - if node_name: - logging.warning(f"JumpCI Pod is running on node/{node_name}") - else: - logging.warning( - "Could not determine Node Name (Check if you are running inside a properly configured K8s Pod)." - ) - - -def entrypoint(ignore_secret_path=False): - apply_preset_from_pr_args = False - - def decorator(fct): - @functools.wraps(fct) - def wrapper(*args, **kwargs): - init(ignore_secret_path, apply_preset_from_pr_args) - return fct(*args, **kwargs) - - return wrapper - - return decorator - - -__keep_open = [] - - -def get_tmp_fd(): - # generate a fd-only temporary file - fd, file_path = tempfile.mkstemp() - - # using only the FD. Ensures that the file disappears when this - # process terminates - os.remove(file_path) - - py_file = os.fdopen(fd, "w") - # this makes sure the FD isn't closed when the var goes out of - # scope - __keep_open.append(py_file) - - return f"/proc/{os.getpid()}/fd/{fd}", py_file - - -def get_lock_owner(): - if os.environ.get("OPENSHIFT_CI") == "true": - build_id = os.environ["BUILD_ID"] - if "PULL_NUMBER" in os.environ: - pr = os.environ["PULL_NUMBER"] - return f"openshift_ci__pr{pr}__build{build_id}" - else: - return f"openshift_ci__periodic__build{build_id}" - else: - logging.warning("Not running in a known CI. Using the username as lock owner.") - return os.environ["USER"] diff --git a/projects/jump_ci/testing/utils_gethostname.py b/projects/jump_ci/testing/utils_gethostname.py deleted file mode 100644 index e9739db1f..000000000 --- a/projects/jump_ci/testing/utils_gethostname.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import logging -import os -import ssl -import urllib.error -import urllib.request - - -def get_kubernetes_node_name() -> str | None: - # generated by Gemini - """ - Retrieves the Kubernetes node name for the current running pod by - communicating with the Kubernetes API server using the pod's service account. - - This function mimics the shell logic: - 1. Read POD_NAME, NAMESPACE, and TOKEN from files/environment. - 2. Construct the APISERVER URL. - 3. Make a secure API call to fetch the pod object. - 4. Extract the '.spec.nodeName' field. - - Returns: - The name of the node the pod is scheduled on, or None if the retrieval fails. - """ - SA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount" - CA_CERT_PATH = os.path.join(SA_PATH, "ca.crt") - - # --- 1. Define Variables --- - - # 1.1 POD_NAME (Hostname) - try: - with open("/proc/sys/kernel/hostname") as f: - POD_NAME = f.read().strip() - except OSError: - # Fallback to os.uname().nodename, which is often the same - POD_NAME = os.uname().nodename - - # 1.2 NAMESPACE - try: - with open(os.path.join(SA_PATH, "namespace")) as f: - NAMESPACE = f.read().strip() - except FileNotFoundError: - logging.error("Namespace file not found. Running outside a standard K8s Pod?") - return None - - # 1.3 TOKEN - try: - with open(os.path.join(SA_PATH, "token")) as f: - TOKEN = f.read().strip() - except FileNotFoundError: - logging.error("Service account token not found.") - return None - - # 1.4 APISERVER URL - K8S_HOST = os.environ.get("KUBERNETES_SERVICE_HOST") - K8S_PORT = os.environ.get("KUBERNETES_SERVICE_PORT") - - if not (K8S_HOST and K8S_PORT): - logging.error("KUBERNETES_SERVICE_HOST or PORT environment variables not set.") - return None - - APISERVER = f"https://{K8S_HOST}:{K8S_PORT}" - - # --- 2. Make the API Call --- - - URL = f"{APISERVER}/api/v1/namespaces/{NAMESPACE}/pods/{POD_NAME}" - - try: - # Create SSL context with CA certificate - ssl_context = ssl.create_default_context(cafile=CA_CERT_PATH) - - # Create request with authorization header - request = urllib.request.Request(URL) - request.add_header("Authorization", f"Bearer {TOKEN}") - - # Make the API call with SSL verification and timeout - with urllib.request.urlopen(request, timeout=5, context=ssl_context) as response: - response_data = response.read() - pod_data = json.loads(response_data.decode("utf-8")) - - # Safely extract the nodeName field - node_name = pod_data.get("spec", {}).get("nodeName") - - if node_name: - return node_name - else: - logging.warning("'nodeName' field not found in the pod specification.") - return None - - except urllib.error.HTTPError as e: - logging.error(f"Failed to query Kubernetes API at {URL}. HTTP Error {e.code}: {e.reason}") - return None - except urllib.error.URLError as e: - logging.error(f"Failed to query Kubernetes API at {URL}. URL Error: {e.reason}") - return None - except json.JSONDecodeError: - logging.error("Failed to parse API response as JSON.") - return None - except OSError as e: - logging.error(f"Failed to read CA certificate or network error: {e}") - return None - - -# --- Example Usage --- -if __name__ == "__main__": - # This block will only run if you execute the script directly - - # NOTE: This uses Python's built-in urllib library (no external dependencies) - - logging.info("Attempting to determine Kubernetes Node Name...") - - NODE_NAME = get_kubernetes_node_name() - - if NODE_NAME: - logging.info(f"\n✅ Pod is running on Node: **{NODE_NAME}**") - else: - logging.info( - "\n❌ Could not determine Node Name (Check if you are running inside a properly configured K8s Pod)." - ) diff --git a/projects/jump_ci/toolbox/jump_ci.py b/projects/jump_ci/toolbox/jump_ci.py deleted file mode 100644 index 3bf3c75f5..000000000 --- a/projects/jump_ci/toolbox/jump_ci.py +++ /dev/null @@ -1,169 +0,0 @@ -from projects.legacy.library.ansible_toolbox import ( - AnsibleGatherFacts, - AnsibleMappedParams, - AnsibleRole, - RunAnsibleRole, -) - - -class Jump_Ci: - """ - Commands to run TOPSAIL scripts in a jump host - """ - - @AnsibleRole("jump_ci_take_lock") - @AnsibleMappedParams - @AnsibleGatherFacts - def take_lock(self, cluster, owner): - """ - Take a lock with a given cluster name on a remote node - - Args: - cluster: name of the cluster lock to take - owner: name of the lock owner - """ - - if not cluster: # don't accept the empty string value - raise ValueError("--cluster must be set") - - return RunAnsibleRole(locals()) - - @AnsibleRole("jump_ci_ensure_lock") - @AnsibleMappedParams - @AnsibleGatherFacts - def ensure_lock(self, cluster, owner, check_kubeconfig=True): - """ - Ensure that cluster lock with a given name is taken. Fails otherwise. - - Args: - cluster: name of the cluster lock to test - owner: name of the lock owner - check_kubeconfig: if enabled, ensure that the cluster's kubeconfig file exists - """ - - if not cluster: # don't accept the empty string value - raise ValueError("--cluster must be set") - - return RunAnsibleRole(locals()) - - @AnsibleRole("jump_ci_release_lock") - @AnsibleMappedParams - @AnsibleGatherFacts - def release_lock(self, cluster, owner): - """ - Release a cluster lock with a given name on a remote node - - Args: - cluster: name of the cluster lock to release - owner: name of the lock owner - """ - - if not cluster: # don't accept the empty string value - raise ValueError("--cluster must be set") - - return RunAnsibleRole(locals()) - - @AnsibleRole("jump_ci_prepare_topsail") - @AnsibleMappedParams - @AnsibleGatherFacts - def prepare_topsail( - self, - cluster, - lock_owner, - pr_number=None, - repo_owner="openshift-psap", - repo_name="topsail-ng", - git_ref=None, - image_name=None, - image_tag=None, - dockerfile_name="projects/core/image/Containerfile", - update_from_imagetag="main", - cleanup_old_pr_images=True, - ): - """ - Prepares the jump host for running TOPSAIL: - - clones TOPSAIL repository - - builds TOPSAIL image in the remote host - - Args: - cluster: Name of the cluster to use - lock_owner: name of the lock owner - pr_number: PR number to use for the test. If none, use the main branch. - - repo_owner: Name of the Github repo owner - repo_name: Name of the TOPSAIL github repo - git_ref: the ref (commit/branch) to use in the git repository. Use the PR's `/merge` if not specify, or the main branch if no PR number is specified. - image_name: Name to use when building TOPSAIL image - image_tag: Name to give to the tag, or computed if empty - dockerfile_name: Name/path of the Dockerfile to use to build the image - cleanup_old_pr_images: if disabled, don't cleanup the old images - update_from_imagetag: if set, update the git tree from this image instead of building from scratch - """ - if not image_name: - image_name = f"localhost/{repo_name}" - - if not cluster: # don't accept the empty string value - raise ValueError("--cluster must be set") - - return RunAnsibleRole(locals()) - - @AnsibleRole("jump_ci_prepare_step") - @AnsibleMappedParams - @AnsibleGatherFacts - def prepare_step( - self, - cluster, - lock_owner, - project, - step, - env_file, - variables_overrides_dict, - secrets_path_env_key=None, - ): - """ - Prepares the jump host for running a CI test step: - - Args: - cluster: Name of the cluster lock to use - lock_owner: name of the lock owner - project: Name of the project to execute - step: Name of the step to execute - env_file: Path to the env file to use - variables_overrides_dict: Dictionnary to save as the variable overrides file - secrets_path_env_key: If provided, the env key will be used to locate the secret directories to upload to the jump host - """ - - if not cluster: # don't accept the empty string value/None - raise ValueError("--cluster must be set") - - if not project: # don't accept the empty string value/None - raise ValueError("--project must be set") - - return RunAnsibleRole(locals()) - - @AnsibleRole("jump_ci_retrieve_artifacts") - @AnsibleMappedParams - @AnsibleGatherFacts - def retrieve_artifacts( - self, - cluster, - lock_owner, - remote_dir, - local_dir="artifacts", - skip_cluster_lock=False, - ): - """ - Prepares the jump host for running a CI test step: - - Args: - cluster: Name of the cluster lock to use - lock_owner: name of the lock owner - remote_dir: name of remote directory to retrieve. - local_dir: name of the local dir where to store the results, within the extra logs artifacts directory. - skip_cluster_lock: if True, skip the cluster is lock check (eg, when included from another role). - """ - - if not cluster: # don't accept the empty string value - raise ValueError("--cluster must be set") - - return RunAnsibleRole(locals()) diff --git a/projects/jump_ci/toolbox/jump_ci_ensure_lock/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_ensure_lock/defaults/main/config.yml deleted file mode 100644 index daad7b9ae..000000000 --- a/projects/jump_ci/toolbox/jump_ci_ensure_lock/defaults/main/config.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.ensure_lock - -# Parameters -# name of the cluster lock to test -# Mandatory value -jump_ci_ensure_lock_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_ensure_lock_owner: - -# if enabled, ensure that the cluster's kubeconfig file exists -jump_ci_ensure_lock_check_kubeconfig: true - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_ensure_lock/files/.keep b/projects/jump_ci/toolbox/jump_ci_ensure_lock/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_ensure_lock/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_ensure_lock/tasks/main.yml deleted file mode 100644 index 2c142d2cc..000000000 --- a/projects/jump_ci/toolbox/jump_ci_ensure_lock/tasks/main.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- name: Log whoami - shell: - whoami - -- name: Ensure that the cluster kubeconfig file exists - when: jump_ci_ensure_lock_check_kubeconfig | bool - block: - - name: Ensure that ansible_env exists - fail: msg="ansible_env does not exists. Make sure that the gather_facts command run. In TOPSAIL, the role needs to run remotely. Or set --check_kubeconfig=false" - when: ansible_env is not defined - - - name: Prepare the cluster kubeconfig path - set_fact: - cluster_kubeconfig: "{{ ansible_env.HOME }}/.kube/config.{{ jump_ci_ensure_lock_cluster }}" - - - name: Ensure that the KUBECONFIG file exists - stat: - path: "{{ cluster_kubeconfig }}" - register: cluster_kubeconfig_stat - - - name: Fail if the cluster's KUBECONFIG file doesn't exist - fail: msg="The cluster KUBECONFIG ({{ cluster_kubeconfig }}) does not exists" - when: not cluster_kubeconfig_stat.stat.exists - -- name: Save the name of the cluster lock directory - set_fact: - cluster_lock_dir: "/tmp/topsail_{{ jump_ci_ensure_lock_cluster }}" - -- name: "Get the stats of the lock directory: {{ jump_ci_ensure_lock_cluster }}" - stat: - path: "{{ cluster_lock_dir }}" - register: lock_directory_stat - -- name: Fail if the lock directory doesn't exist - fail: msg="The lock directory '{{ cluster_lock_dir }}' does not exists" - when: not lock_directory_stat.stat.exists - -- name: Get the owner of the lock - command: - cat "{{ cluster_lock_dir }}/lock_owner" - register: lock_owner_content_cmd - -- name: Fail if the owner is different - when: lock_owner_content_cmd.stdout | trim != jump_ci_ensure_lock_owner - fail: msg="Unexpected lock owner. Found '{{ lock_owner_content_cmd.stdout }}', expected '{{ jump_ci_ensure_lock_owner }}'" diff --git a/projects/jump_ci/toolbox/jump_ci_ensure_lock/templates/.keep b/projects/jump_ci/toolbox/jump_ci_ensure_lock/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_ensure_lock/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_ensure_lock/vars/main/resources.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_prepare_step/defaults/main/config.yml deleted file mode 100644 index 5ea0efe8c..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_step/defaults/main/config.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.prepare_step - -# Parameters -# Name of the cluster lock to use -# Mandatory value -jump_ci_prepare_step_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_prepare_step_lock_owner: - -# Name of the project to execute -# Mandatory value -jump_ci_prepare_step_project: - -# Name of the step to execute -# Mandatory value -jump_ci_prepare_step_step: - -# Path to the env file to use -# Mandatory value -jump_ci_prepare_step_env_file: - -# Dictionnary to save as the variable overrides file -# Mandatory value -jump_ci_prepare_step_variables_overrides_dict: - -# If provided, the env key will be used to locate the secret directories to upload to the jump host -jump_ci_prepare_step_secrets_path_env_key: null - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/files/.keep b/projects/jump_ci/toolbox/jump_ci_prepare_step/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_prepare_step/tasks/main.yml deleted file mode 100644 index 69446b71d..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_step/tasks/main.yml +++ /dev/null @@ -1,104 +0,0 @@ ---- -- name: Ensure that the lock is owned - include_role: - name: jump_ci_ensure_lock - # sets cluster_kubeconfig - vars: - jump_ci_ensure_lock_cluster: "{{ jump_ci_prepare_step_cluster }}" - jump_ci_ensure_lock_owner: "{{ jump_ci_prepare_step_lock_owner }}" - -- name: Fetch the name of the image to use - shell: - cat "{{ cluster_lock_dir }}/pod.image" - register: pod_image_cmd - failed_when: false - -- name: Fail if the image file is not set - when: pod_image_cmd.rc != 0 - fail: msg="The pod image isn't prepared. Did the 'prepare_topsail' step run?" - -- name: Generate the step artifacts dirname - set_fact: - step_artifact_dir: "{{ cluster_lock_dir }}/test/{{ jump_ci_prepare_step_step }}" - -- name: Ensure that the step artifacts dirname directory is empty - ansible.builtin.file: - state: absent - path: "{{ step_artifact_dir }}" - mode: '0755' - no_log: true # can be too chatty when cleaning an existing artifacts directly - -- name: Ensure that the step test dirname directory exists - ansible.builtin.file: - state: directory - path: "{{ step_artifact_dir }}" - mode: '0755' - -- name: Ensure that the step test artifacts dirname directory exists - ansible.builtin.file: - state: directory - path: "{{ step_artifact_dir }}/artifacts" - mode: '0755' - -- name: Copy the env file - ansible.builtin.copy: - src: "{{ jump_ci_prepare_step_env_file }}" - dest: "{{ step_artifact_dir }}/test_env" - mode: '0644' - -- name: Copy all the jump-host env file locally - ansible.posix.synchronize: - src: "{{ step_artifact_dir }}/test_env" - dest: "{{ artifact_extra_logs_dir }}/jump_host.env" - mode: "pull" - no_log: true # not necessary, and leaks the hostnames - -- name: Populate the variables_overrides files - shell: | - mkdir -p {{ step_artifact_dir }}/artifacts/000__ci_metadata; - cat <> {{ step_artifact_dir }}/artifacts/000__ci_metadata/variable_overrides.yaml - {{ (jump_ci_prepare_step_variables_overrides_dict or {}) | to_nice_yaml }} - EOF - -- name: Copy all the jump-host env file locally - ansible.posix.synchronize: - src: "{{ step_artifact_dir }}/artifacts/000__ci_metadata/variable_overrides.yaml" - dest: "{{ artifact_extra_logs_dir }}/variable_overrides.yaml" - mode: "pull" - no_log: true # not necessary, and leaks the hostnames - -- name: Ensure that the secrets directory exists - ansible.builtin.file: - state: directory - path: "{{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}" - mode: '0755' - -- name: Copy the secrets to the jump host - ansible.posix.synchronize: - src: "{{ lookup('env', jump_ci_prepare_step_secrets_path_env_key) }}/" - dest: "{{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}/" - mode: "push" - no_log: true # not necessary, and leaks the hostnames and secret filenames (they aren't secret) - when: jump_ci_prepare_step_secrets_path_env_key is defined and (jump_ci_prepare_step_secrets_path_env_key | string) != "" - -- name: Ensure that the secret dirs have the right permission - command: find "{{ cluster_lock_dir }}/secrets" -type d -exec chmod 0755 {} \; - -- name: Ensure that the secret files have the right permission - command: find "{{ cluster_lock_dir }}/secrets/" -type f -exec chmod 0400 {} \; - -- name: Prepare the entrypoint script - template: - src: "{{ entrypoint_template }}" - dest: "{{ step_artifact_dir }}/entrypoint.sh" - mode: '0400' - -- name: Copy all the entrypoint file locally - ansible.posix.synchronize: - src: "{{ step_artifact_dir }}/entrypoint.sh" - dest: "{{ artifact_extra_logs_dir }}/entrypoint.sh" - mode: "pull" - no_log: true # not necessary, and leaks the hostnames - -- name: Show the launch command - debug: msg="run 'bash {{ step_artifact_dir }}/entrypoint.sh' in the remote host" diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/templates/.keep b/projects/jump_ci/toolbox/jump_ci_prepare_step/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/templates/entrypoint.sh.j2 b/projects/jump_ci/toolbox/jump_ci_prepare_step/templates/entrypoint.sh.j2 deleted file mode 100644 index a03335737..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_step/templates/entrypoint.sh.j2 +++ /dev/null @@ -1,68 +0,0 @@ -#! /bin/bash - -set -o pipefail -set -o errexit -set -o nounset -set -o errtrace - -interactive="" -cmd="set -a; source /run/secrets/test_env; set +a; exec /app/forge/bin/run_ci '{{ jump_ci_prepare_step_project }}' ci '{{ jump_ci_prepare_step_step }}'" -image="{{ pod_image_cmd.stdout | trim}}" - -cleanup() { - echo "Stopping the container..." - podman rm --force --ignore forge-on-{{ jump_ci_prepare_step_cluster }} - - # Not cleaning up the secrets for now. They are safe anyway - # echo Cleaning up the secrets .. - # rm -rf {{ cluster_lock_dir }}/secrets -} - -# --- - -if [[ "${1:-}" == enter ]]; then - echo "Entering the container instead of running '$cmd'" - interactive="-it" - cmd="bash" -fi - -if [[ "${1:-}" == cleanup ]]; then - echo "Running the container cleanup ..." - cleanup - exit 0 -fi - -trap cleanup INT TERM ERR - -# --- - -# makes it easier to rerun manually after a failure -rm -f "{{ step_artifact_dir }}/artifacts/FAILURES" - -ARTIFACT_DIR="/log/artifacts" - -echo "### Running '$cmd' against image '$image' ###" - -podman run \ - --userns=keep-id \ - --user "$(id -u):$(id -g)" \ - --rm \ - --name forge-on-{{ jump_ci_prepare_step_cluster }} \ - \ - --volume "{{ step_artifact_dir }}/artifacts:$ARTIFACT_DIR:Z" \ - --env ARTIFACT_DIR="$ARTIFACT_DIR" \ - \ - --volume {{ cluster_lock_dir }}/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:/run/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}:ro \ - --env {{ jump_ci_prepare_step_secrets_path_env_key }}="/run/secrets/{{ jump_ci_prepare_step_secrets_path_env_key }}" \ - \ -{%- if cluster_kubeconfig is defined -%} - --volume {{ cluster_kubeconfig }}:/run/secrets/kubeconfig:ro \ -{%- endif -%} - --env KUBECONFIG="/run/secrets/kubeconfig" \ - \ - --volume {{ step_artifact_dir }}/test_env:/run/secrets/test_env:ro \ - $interactive \ - "$image" \ - bash -ce "$cmd" |& tee -a "{{ step_artifact_dir }}/run.log" - -cleanup diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_step/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_prepare_step/vars/main/resources.yml deleted file mode 100644 index 90619931d..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_step/vars/main/resources.yml +++ /dev/null @@ -1 +0,0 @@ -entrypoint_template: templates/entrypoint.sh.j2 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_prepare_topsail/defaults/main/config.yml deleted file mode 100644 index 720422dd5..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/defaults/main/config.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.prepare_topsail - -# Parameters -# Name of the cluster to use -# Mandatory value -jump_ci_prepare_topsail_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_prepare_topsail_lock_owner: - -# PR number to use for the test. If none, use the main branch. -jump_ci_prepare_topsail_pr_number: null - -# Name of the Github repo owner -jump_ci_prepare_topsail_repo_owner: openshift-psap - -# Name of the TOPSAIL github repo -jump_ci_prepare_topsail_repo_name: topsail - -# the ref (commit/branch) to use in the git repository. Use the PR's `/merge` if not specify, or the main branch if no PR number is specified. -jump_ci_prepare_topsail_git_ref: null - -# Name to use when building TOPSAIL image -jump_ci_prepare_topsail_image_name: localhost/topsail - -# Name to give to the tag, or computed if empty -jump_ci_prepare_topsail_image_tag: null - -# Name/path of the Dockerfile to use to build the image -jump_ci_prepare_topsail_dockerfile_name: build/Dockerfile - -# if set, update the git tree from this image instead of building from scratch -jump_ci_prepare_topsail_update_from_imagetag: main - -# if disabled, don't cleanup the old images -jump_ci_prepare_topsail_cleanup_old_pr_images: true - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/files/.keep b/projects/jump_ci/toolbox/jump_ci_prepare_topsail/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_prepare_topsail/tasks/main.yml deleted file mode 100644 index 73820462e..000000000 --- a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/tasks/main.yml +++ /dev/null @@ -1,196 +0,0 @@ ---- -- name: Ensure that the lock is owned - include_role: - name: jump_ci_ensure_lock - vars: - jump_ci_ensure_lock_cluster: "{{ jump_ci_prepare_topsail_cluster }}" - jump_ci_ensure_lock_owner: "{{ jump_ci_prepare_topsail_lock_owner }}" - -- name: Generate the artifact dirname - set_fact: - artifact_dir: "{{ cluster_lock_dir }}/jump_ci_artifacts" - -- name: Ensure that the logs directory exists - ansible.builtin.file: - state: directory - path: "{{ artifact_dir }}" - mode: '0755' - -- name: Compute the index of the next logs directory - shell: - set -o pipefail; - printf "%03d" $(ls "{{ artifact_dir }}/0*" -d 2>/dev/null | wc -l) - register: next_artifact_dir_count - -- name: Set the logs directory name - set_fact: - artifact_extra_logs_dir: "{{ artifact_dir }}/{{ next_artifact_dir_count.stdout }}__jump_ci__prepare_topsail" - -- name: Ensure that the top artifacts directory exists - ansible.builtin.file: - state: directory - path: "{{ artifact_extra_logs_dir }}" - mode: '0755' - -- name: Ensure that the artifacts directory exists - ansible.builtin.file: - state: directory - path: "{{ artifact_extra_logs_dir }}/artifacts" - mode: '0755' - -- name: Ensure that the src directory exists - ansible.builtin.file: - state: directory - path: "{{ artifact_extra_logs_dir }}/src" - mode: '0755' - -- name: Show the artifacts dir - debug: msg="Using '{{ artifact_extra_logs_dir }}' as artifacts directory" - -- name: Set TOPSAIL's directory - set_fact: - topsail_home: "{{ cluster_lock_dir }}/forge" - -- name: Ensure that the src directory exists - ansible.builtin.file: - state: absent - path: "{{ topsail_home }}" - -- name: Set git command - set_fact: - git: git -C "{{ topsail_home }}" - git_src: "{% if jump_ci_prepare_topsail_pr_number %}pr-{{ jump_ci_prepare_topsail_pr_number }}{% else %}main{% endif %}" - -- name: Git checkout - ansible.builtin.git: - repo: 'https://github.com/{{ jump_ci_prepare_topsail_repo_owner }}/{{ jump_ci_prepare_topsail_repo_name }}.git' - dest: "{{ topsail_home }}" - # always clone the main branch at this stage. The exact commit will be fetched next. - version: "main" - force: true - -- name: Save git ref - set_fact: - git_ref: >- - {% if jump_ci_prepare_topsail_git_ref -%} - {{ jump_ci_prepare_topsail_git_ref }} - {%- elif jump_ci_prepare_topsail_pr_number is defined and (jump_ci_prepare_topsail_pr_number | string) != "" -%} - refs/pull/{{ jump_ci_prepare_topsail_pr_number }}/merge - {%- else -%} - main - {%- endif -%} - -- name: Fetch TOPSAIL repository - block: - - name: Fetch TOPSAIL's branch - command: | - {{ git }} fetch --quiet origin {{ git_ref }} - - when: jump_ci_prepare_topsail_pr_number is defined and (jump_ci_prepare_topsail_pr_number | string) != "" - - - name: Switch to TOPSAIL's branch - command: | - {{ git }} reset --hard FETCH_HEAD - - when: jump_ci_prepare_topsail_pr_number is defined and (jump_ci_prepare_topsail_pr_number | string) != "" - - - name: Fetch the submodules - command: | - {{ git }} submodule --quiet update --init --recursive --force - - - name: Show the submodules version - command: | - {{ git }} submodule - -- name: Get the relevant git commit - shell: | - {{ git }} rev-parse HEAD - register: git_commit_cmd - -- name: Show the commit - shell: | - set -o pipefail; - {{ git }} show --quiet {{ git_commit_cmd.stdout }} | tee "{{ artifact_extra_logs_dir }}/src/topsail.git-version" - -- name: Generate the image tag - set_fact: - image_tag: "{% if jump_ci_prepare_topsail_image_tag %}{{ jump_ci_prepare_topsail_image_tag }}{% else %}{{ git_src }}-{{ git_commit_cmd.stdout[:9] }}{% endif %}" - -- name: Generate the image full name - set_fact: - image_full_name: "{{ jump_ci_prepare_topsail_image_name }}:{{ image_tag }}" - -- name: Check if the image tag already exists - shell: - podman inspect {{ image_full_name }} > /dev/null - && echo "Image '{{ image_full_name }}' already exists. No need to rebuild it." - register: has_image - failed_when: false - -- name: List all the old images to delete - # setting --filter label=preserve!=true in Podman args doesn't work - shell: - set -o pipefail; - podman image ls - --filter until=4h - --filter label=name=topsail-ng - --format json - | jq -r '.[] | select(.Containers == 0) | select(.Labels.preserve != "true") | .Id' - register: podman_images_to_delete_cmd - -# To add the 'preserve=true' label: -# -# CONTAINER_NAME=localhost/topsail:main -# LABEL="preserve=true" -# echo "FROM $CONTAINER_NAME" | podman build --label "$LABEL" -t "$CONTAINER_NAME" - - -- name: Prune old images - command: - podman image rm "{{ image_id }}" - when: - - jump_ci_prepare_topsail_cleanup_old_pr_images - - has_image.rc != 0 - loop: "{{ podman_images_to_delete_cmd.stdout_lines }}" - loop_control: - loop_var: image_id - # I have seen some unpected failure because of (dependency) images used by a container (not sure which one :/) - # so ignoring this failure for the time being. - failed_when: false - -- name: "Build image as an update {{ image_full_name }}" - shell: | - cat > {{ cluster_lock_dir }}/overlay.containerfile < 0 - -- name: "Build image from scatch {{ image_full_name }}" - shell: - time podman build --squash - --label owner=forge - --tag "{{ image_full_name }}" - --file "{{ jump_ci_prepare_topsail_dockerfile_name }}" - "{{ topsail_home }}" > "{{ artifact_extra_logs_dir }}/artifacts/image.log" - when: - - has_image.rc != 0 - - jump_ci_prepare_topsail_update_from_imagetag | default('', true) | length == 0 - -- name: Save the name of the image to use - shell: - echo "{{ image_full_name }}" > "{{ cluster_lock_dir }}/pod.image" diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/templates/.keep b/projects/jump_ci/toolbox/jump_ci_prepare_topsail/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_prepare_topsail/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_prepare_topsail/vars/main/resources.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_release_lock/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_release_lock/defaults/main/config.yml deleted file mode 100644 index 617a4ce1e..000000000 --- a/projects/jump_ci/toolbox/jump_ci_release_lock/defaults/main/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.release_lock - -# Parameters -# name of the cluster lock to release -# Mandatory value -jump_ci_release_lock_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_release_lock_owner: - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_release_lock/files/.keep b/projects/jump_ci/toolbox/jump_ci_release_lock/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_release_lock/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_release_lock/tasks/main.yml deleted file mode 100644 index 0da2bb523..000000000 --- a/projects/jump_ci/toolbox/jump_ci_release_lock/tasks/main.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Ensure that the lock is owned - include_role: - name: jump_ci_ensure_lock - vars: - jump_ci_ensure_lock_cluster: "{{ jump_ci_release_lock_cluster }}" - jump_ci_ensure_lock_owner: "{{ jump_ci_release_lock_owner }}" - jump_ci_ensure_lock_check_kubeconfig: false - -- name: Release the lock directory - command: - rm -rf "{{ cluster_lock_dir }}" diff --git a/projects/jump_ci/toolbox/jump_ci_release_lock/templates/.keep b/projects/jump_ci/toolbox/jump_ci_release_lock/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_release_lock/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_release_lock/vars/main/resources.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/defaults/main/config.yml deleted file mode 100644 index 15e1f1fe5..000000000 --- a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/defaults/main/config.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.retrieve_artifacts - -# Parameters -# Name of the cluster lock to use -# Mandatory value -jump_ci_retrieve_artifacts_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_retrieve_artifacts_lock_owner: - -# name of remote directory to retrieve. -# Mandatory value -jump_ci_retrieve_artifacts_remote_dir: - -# name of the local dir where to store the results, within the extra logs artifacts directory. -jump_ci_retrieve_artifacts_local_dir: artifacts - -# if True, skip the cluster is lock check (eg, when included from another role). -jump_ci_retrieve_artifacts_skip_cluster_lock: false - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/files/.keep b/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/tasks/main.yml deleted file mode 100644 index 635e6d84c..000000000 --- a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/tasks/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Ensure that the lock is owned - include_role: - name: jump_ci_ensure_lock - vars: - jump_ci_ensure_lock_cluster: "{{ jump_ci_retrieve_artifacts_cluster }}" - jump_ci_ensure_lock_owner: "{{ jump_ci_retrieve_artifacts_lock_owner }}" - when: not jump_ci_retrieve_artifacts_skip_cluster_lock - -- name: Compute the path of the directory to retrieve - set_fact: - remote_dir_path: "{% if cluster_lock_dir is defined %}{{ cluster_lock_dir }}{% else %}{{ '/tmp/topsail_' + jump_ci_retrieve_artifacts_cluster }}{% endif %}/{{ jump_ci_retrieve_artifacts_remote_dir }}" - -- name: Get the size of the directory to retrieve - command: - du -sh "{{ remote_dir_path }}" - -- name: Copy all the files locally - ansible.posix.synchronize: - src: "{{ remote_dir_path }}/" - dest: "{{ artifact_extra_logs_dir }}/{{ jump_ci_retrieve_artifacts_local_dir }}" - mode: "pull" - recursive: true - no_log: true # too verbose diff --git a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/templates/.keep b/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_retrieve_artifacts/vars/main/resources.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_take_lock/defaults/main/config.yml b/projects/jump_ci/toolbox/jump_ci_take_lock/defaults/main/config.yml deleted file mode 100644 index 0083d374d..000000000 --- a/projects/jump_ci/toolbox/jump_ci_take_lock/defaults/main/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Auto-generated file, do not edit manually ... -# Toolbox generate command: repo generate_ansible_default_settings -# Source component: Jump_Ci.take_lock - -# Parameters -# name of the cluster lock to take -# Mandatory value -jump_ci_take_lock_cluster: - -# name of the lock owner -# Mandatory value -jump_ci_take_lock_owner: - -# Default Ansible variables -# Default value for ansible_os_family to ensure role remains standalone -ansible_os_family: Linux diff --git a/projects/jump_ci/toolbox/jump_ci_take_lock/files/.keep b/projects/jump_ci/toolbox/jump_ci_take_lock/files/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_take_lock/tasks/main.yml b/projects/jump_ci/toolbox/jump_ci_take_lock/tasks/main.yml deleted file mode 100644 index 27813bd51..000000000 --- a/projects/jump_ci/toolbox/jump_ci_take_lock/tasks/main.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: Log whoami - shell: - whoami - -- name: Save the name of the cluster lock directory - set_fact: - cluster_lock_dir: "/tmp/topsail_{{ jump_ci_take_lock_cluster }}" - -- name: Create the lock directory - shell: | - if mkdir "{{ cluster_lock_dir }}" 2>/dev/null; then - echo "$(date) Lock '{{ jump_ci_take_lock_cluster }}' acquired."; - exit 0 - fi - - echo "$(date) Lock '{{ jump_ci_take_lock_cluster }}' already taken ..." - cat "{{ cluster_lock_dir }}/lock_owner" 2>/dev/null || true - exit 1 - - register: lock_directory_created_cmd - until: lock_directory_created_cmd.rc == 0 - retries: 9999 - delay: 10 - -- name: Log info about the lock owner - shell: - echo "{{ jump_ci_take_lock_owner }}" > "{{ cluster_lock_dir }}/lock_owner" - -- name: Ensure that the lock is owned - include_role: - name: jump_ci_ensure_lock - vars: - jump_ci_ensure_lock_cluster: "{{ jump_ci_take_lock_cluster }}" - jump_ci_ensure_lock_owner: "{{ jump_ci_take_lock_owner }}" diff --git a/projects/jump_ci/toolbox/jump_ci_take_lock/templates/.keep b/projects/jump_ci/toolbox/jump_ci_take_lock/templates/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/jump_ci/toolbox/jump_ci_take_lock/vars/main/resources.yml b/projects/jump_ci/toolbox/jump_ci_take_lock/vars/main/resources.yml deleted file mode 100644 index e69de29bb..000000000 From 95d276deb948871376deabde7d958f8c89d0db37 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:21:20 +0200 Subject: [PATCH 0881/1072] [repo] toolbox: repo: remove --- .github/workflows/check_consistency.yml | 3 - projects/repo/toolbox/repo.py | 154 ------------------------ 2 files changed, 157 deletions(-) delete mode 100644 projects/repo/toolbox/repo.py diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index e447d86e9..c040fc86a 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -25,9 +25,6 @@ jobs: - uses: ./.github/steps/prepare_minimal_ubuntu_image - - name: Check that no symlink is broken - run: bin/run_ansible_toolbox.py repo validate_no_broken_link - - name: Check that all source files end with EOL run: | # Find all source files and check they end with newline diff --git a/projects/repo/toolbox/repo.py b/projects/repo/toolbox/repo.py deleted file mode 100644 index fd96cc4a7..000000000 --- a/projects/repo/toolbox/repo.py +++ /dev/null @@ -1,154 +0,0 @@ -import json -import logging -import os -import pathlib -import subprocess -import sys -import urllib.error -import urllib.request - -TOOLBOX_THIS_DIR = pathlib.Path(__file__).absolute().parent -PROJECT_DIR = TOOLBOX_THIS_DIR.parent - -logger = logging.getLogger(__name__) - - -class Repo: - """ - Commands to perform consistency validations on this repo itself - """ - - @staticmethod - def validate_no_wip(): - """ - Ensures that none of the commits have the WIP flag in their - message title. - """ - WIP_MARKER = "WIP" - - # Check if running in GitHub Actions - github_ref = os.environ.get("GITHUB_REF") - if not github_ref: - logger.error("GITHUB_REF not set, cannot run outside of a GitHub action.") - sys.exit(1) - - github_repository = os.environ.get("GITHUB_REPOSITORY") - if not github_repository: - logger.error("GITHUB_REPOSITORY not set.") - sys.exit(1) - - try: - # Extract PR number from GITHUB_REF (format: refs/pull/123/merge) - pr_number = github_ref.split("/")[2] - pr_url = f"https://api.github.com/repos/{github_repository}/pulls/{pr_number}" - - logger.info(f"Fetching the PR from '{pr_url}' ...") - - # Fetch PR data from GitHub API - with urllib.request.urlopen(pr_url) as response: - pr_data = json.loads(response.read().decode()) - - pr_title = pr_data["title"] - logger.info(f"PR title: {pr_title}") - - # Get commit messages using git - try: - # Get first and second parent from latest commit - result = subprocess.run( - ["git", "log", "--pretty=%P", "-n", "1"], - capture_output=True, - text=True, - check=True, - ) - parents = result.stdout.strip().split() - - if len(parents) >= 2: - first_parent = parents[0] - second_parent = parents[1] - - # Get commits between the parents - result = subprocess.run( - [ - "git", - "log", - "--pretty=format:%h - %s", - "--abbrev-commit", - f"{first_parent}..{second_parent}", - ], - capture_output=True, - text=True, - check=True, - ) - commits = result.stdout.strip() - - logger.info("PR commits:") - logger.info(commits) - else: - commits = "" - logger.info("No merge commits found") - - except subprocess.CalledProcessError as e: - logger.error(f"Failed to get git commits: {e}") - sys.exit(1) - - # Check for WIP marker in PR title - if WIP_MARKER in pr_title: - logger.error(f"Found the '{WIP_MARKER}' marker in the PR title") - sys.exit(1) - - # Check for WIP marker in commits - if commits and WIP_MARKER in commits: - logger.error(f"Found the '{WIP_MARKER}' marker in the PR commits") - sys.exit(1) - - logger.info("No WIP markers found") - sys.exit(0) - - except urllib.error.HTTPError as e: - logger.error(f"Failed to fetch PR data: {e}") - sys.exit(1) - except (KeyError, IndexError, ValueError) as e: - logger.error(f"Failed to parse GitHub data: {e}") - sys.exit(1) - - @staticmethod - def validate_no_broken_link(): - """ - Ensure that all the symlinks point to a file - """ - broken_links = [] - - def check_directory(directory): - """Recursively check directory for broken symlinks""" - try: - for item in directory.iterdir(): - if item.is_symlink(): - # Check if the symlink target exists - if not item.exists(): - broken_links.append(item) - elif item.is_dir(): - # Recursively check subdirectories - try: - check_directory(item) - except PermissionError: - # Skip directories we can't access - pass - except PermissionError: - # Skip directories we can't access - pass - - logger.info("Checking for broken symlinks...") - check_directory(pathlib.Path(".")) - - if broken_links: - logger.error("Found broken symlinks:") - for link in broken_links: - try: - target = link.readlink() - logger.error(f" {link} -> {target} (broken)") - except OSError as e: - logger.error(f" {link} (error reading target: {e})") - sys.exit(1) - else: - logger.info("No broken symlinks found") - sys.exit(0) From 7f79992c19cab653c676f60d7990b0a1a06332f0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 11:23:05 +0200 Subject: [PATCH 0882/1072] Remove TOPSAIL dependencies on fire and ansible --- .github/steps/prepare_minimal_ubuntu_image/action.yml | 2 +- docs/ci/run_ci.md | 2 +- docs/toolbox/toolbox-command-guide.md | 1 - projects/core/image/pyproject-lightweight.toml | 1 - pyproject.toml | 2 -- 5 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/steps/prepare_minimal_ubuntu_image/action.yml b/.github/steps/prepare_minimal_ubuntu_image/action.yml index 4570b25fc..fcefcc61f 100644 --- a/.github/steps/prepare_minimal_ubuntu_image/action.yml +++ b/.github/steps/prepare_minimal_ubuntu_image/action.yml @@ -7,4 +7,4 @@ runs: - name: Install dependencies shell: bash run: | - python3 -m pip install fire pyyaml jsonpath_ng joblib + python3 -m pip install pyyaml jsonpath_ng diff --git a/docs/ci/run_ci.md b/docs/ci/run_ci.md index 037aeabfb..b2e24112e 100644 --- a/docs/ci/run_ci.md +++ b/docs/ci/run_ci.md @@ -56,7 +56,7 @@ Scripts must be executable Python files that follow Click CLI conventions and mu **prepare_ci.py** performs comprehensive environment setup: - **Artifact Directory Management**: Creates and validates `$ARTIFACT_DIR` for storing CI artifacts -- **Dependency Installation**: Automatically installs required packages (`fire`, `click`) using `uv` or `pip` +- **Dependency Installation**: Automatically installs required packages (`click`) using `uv` or `pip` - **Tool Validation**: Ensures required tools (`jq`, `oc` for full images) are available - **Environment Variables**: Sets up `FORGE_HOME` and other necessary variables diff --git a/docs/toolbox/toolbox-command-guide.md b/docs/toolbox/toolbox-command-guide.md index 37c15b24c..4814697cb 100644 --- a/docs/toolbox/toolbox-command-guide.md +++ b/docs/toolbox/toolbox-command-guide.md @@ -779,7 +779,6 @@ When writing a toolbox command, ensure: - **Specs**: Check `specs/008-toolbox-dsl/` for detailed DSL documentation - **Examples**: Look at existing commands in `projects/*/toolbox/` -- **TOPSAIL Reference**: Check [openshift-psap/topsail](https://github.com/openshift-psap/topsail/pulls/@me) for mature patterns - **Code Review**: Have colleagues review for adherence to these principles Remember: The toolbox is a **DSL**, not plain Python. Embrace the constraints - they make your commands more reliable, debuggable, and reusable! diff --git a/projects/core/image/pyproject-lightweight.toml b/projects/core/image/pyproject-lightweight.toml index cff5fe8c8..5353732a1 100644 --- a/projects/core/image/pyproject-lightweight.toml +++ b/projects/core/image/pyproject-lightweight.toml @@ -8,7 +8,6 @@ dependencies = [ "yq", "pyyaml", "click>=8.1.0", - "ansible", "fire", "jinja2>=3.1.0,<3.2.0", "slack_sdk==3.35.0", "requests", diff --git a/pyproject.toml b/pyproject.toml index 18b65ee01..51f7237de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,12 +33,10 @@ dependencies = [ "click>=8.1.0", "requests", "slack_sdk==3.35.0", - "ansible", "fire", "pyjwt", "jsonpath_ng", "pydantic>=2.0", "yq", - "joblib", ] [project.optional-dependencies] From 2514d57ad0613604008f14872ea69d978d71458a Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 12:01:14 +0200 Subject: [PATCH 0883/1072] [core] Finish updating the get_ci_metadata_dir function --- projects/core/agentic/config_review/__init__.py | 6 +++--- projects/core/agentic/on_failure/__init__.py | 2 +- projects/core/library/ci.py | 5 ++++- projects/core/library/export.py | 2 +- projects/core/notifications/send.py | 6 +++--- projects/fournos_launcher/orchestration/submit.py | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/projects/core/agentic/config_review/__init__.py b/projects/core/agentic/config_review/__init__.py index 08fb43be2..fd9c7bcba 100644 --- a/projects/core/agentic/config_review/__init__.py +++ b/projects/core/agentic/config_review/__init__.py @@ -131,8 +131,8 @@ def load_artifact_directory_data(base_artifact_dir: str | Path) -> dict[str, Any # Define required files required_files = { "config": base_artifact_dir / "config.yaml", - "presets_applied": ci_lib.get_ci_metadata_dir() / "presets_applied.txt", - "fournos_fjob": ci_lib.get_ci_metadata_dir() / "fournos_fjob.yaml", + "presets_applied": ci_lib.get_ci_metadata_dir_location() / "presets_applied.txt", + "fournos_fjob": ci_lib.get_ci_metadata_dir_location() / "fournos_fjob.yaml", } result = { @@ -319,7 +319,7 @@ def analyze_test_with_llm( # Save to 000__ci_metadata/notifications/000__TEST_DESCRIPTION.txt try: if base_artifact_dir: - notifications_dir = ci_lib.get_ci_metadata_dir() / "notifications" + notifications_dir = ci_lib.get_ci_metadata_dir_location() / "notifications" notifications_dir.mkdir(parents=True, exist_ok=True) test_description_path = notifications_dir / "000__TEST_DESCRIPTION.txt" else: diff --git a/projects/core/agentic/on_failure/__init__.py b/projects/core/agentic/on_failure/__init__.py index 0c9ee8936..a4d577efe 100755 --- a/projects/core/agentic/on_failure/__init__.py +++ b/projects/core/agentic/on_failure/__init__.py @@ -341,7 +341,7 @@ def _generate_unique_failure_review_path(base_artifact_dir: Path, failure_dir_na Unique path for the FAILURE_REVIEW file in 000__ci_metadata/notifications/ """ # Create notifications directory - notifications_dir = ci_lib.get_ci_metadata_dir() / "notifications" + notifications_dir = ci_lib.get_ci_metadata_dir_location() / "notifications" notifications_dir.mkdir(parents=True, exist_ok=True) base_filename = f"090__FAILURE_REVIEW_{failure_dir_name}.txt" diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 0e1615d3d..8457830e1 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -21,7 +21,10 @@ def get_ci_metadata_dir_location(): - return env.BASE_ARTIFACT_DIR / "000__ci_metadata" + metadata_dir = env.BASE_ARTIFACT_DIR / "000__ci_metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + + return metadata_dir # CI metadata directory path diff --git a/projects/core/library/export.py b/projects/core/library/export.py index b1a545d57..92af75c28 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -96,7 +96,7 @@ def _update_fjob_export_status(status: ExportStatus): # Save a copy of the updated fjob to metadata directory try: - metadata_dir = ci_lib.get_ci_metadata_dir() + metadata_dir = ci_lib.get_ci_metadata_dir_location() fournos_fjob_path = metadata_dir / "fournos_fjob.yaml" fournos_fjob_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 267e9847f..db1f1d9ac 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -216,7 +216,7 @@ def _get_notification_content(artifact_dir: pathlib.Path, get_link, get_bold) -> Returns: Formatted notification content string """ - notifications_dir = ci_lib.get_ci_metadata_dir() / "notifications" + notifications_dir = ci_lib.get_ci_metadata_dir_location() / "notifications" failures_file = artifact_dir / "FAILURES.txt" # Guard: Check if notifications directory exists @@ -397,7 +397,7 @@ def get_common_message(finish_reason: str, status: str, get_link, get_italics, g except Exception as e: logger.warning("Failed to read NOTIFICATION-github.md: %s", e) - if (var_over := ci_lib.get_ci_metadata_dir() / "pr_config.txt").exists(): + if (var_over := ci_lib.get_ci_metadata_dir_location() / "pr_config.txt").exists(): with open(var_over) as f: message += f""" {get_bold("Test configuration")}: @@ -405,7 +405,7 @@ def get_common_message(finish_reason: str, status: str, get_link, get_italics, g {f.read().strip()} ``` """ - elif (var_over := ci_lib.get_ci_metadata_dir() / "variable_overrides.yaml").exists(): + elif (var_over := ci_lib.get_ci_metadata_dir_location() / "variable_overrides.yaml").exists(): with open(var_over) as f: message += f""" {get_bold("Test configuration")}: diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index b05044a31..c93b325cb 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -110,7 +110,7 @@ def send_github_notification( # Read test configuration from pr_config.txt pr_config_content = "Configuration not available" - metadata_dir = ci_lib.get_ci_metadata_dir() + metadata_dir = ci_lib.get_ci_metadata_dir_location() pr_config_file = metadata_dir / "pr_config.txt" if pr_config_file.exists(): try: From d54a47a36f516f4506cb30639098a8666fa6774e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 12:40:35 +0200 Subject: [PATCH 0884/1072] [core] notifications: send: update the CI base link --- projects/core/notifications/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index db1f1d9ac..65bf31c57 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -584,7 +584,7 @@ def get_ci_base_link(is_raw_file=False, is_dir=False): link_path = f"pr-logs/pull/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}" return ( - "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/" + "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results-public/" + link_path + f"/artifacts/{test_name}/{test_path}", "", From a96e1d79f88217817a17e6cda9fedaa5436ed0dd Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 14:02:28 -0400 Subject: [PATCH 0885/1072] gitops: add Intlab Forge overlay --- fournos/gitops/README.md | 21 +++++++++- .../applications/forge-intlab-production.yaml | 39 +++++++++++++++++++ .../intlab-production/kustomization.yaml | 32 +++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 fournos/gitops/applications/forge-intlab-production.yaml create mode 100644 fournos/gitops/overlays/intlab-production/kustomization.yaml diff --git a/fournos/gitops/README.md b/fournos/gitops/README.md index 4eae41404..b71a73dc2 100644 --- a/fournos/gitops/README.md +++ b/fournos/gitops/README.md @@ -26,7 +26,9 @@ gitops/ └── overlays/ # Environment-specific configurations ├── development/ │ └── kustomization.yaml - └── production/ + ├── production/ + │ └── kustomization.yaml + └── intlab-production/ └── kustomization.yaml ``` @@ -96,6 +98,20 @@ spec: - Manual approval recommended for critical changes - Extended revision history +### Intlab production (`psap-automation` namespace) + +The Intlab cluster does not run the OpenShift internal image registry. Use +`overlays/intlab-production`, which keeps the shared Forge manifests but points +all Fournos resolver Jobs at `quay.io/rh_perfscale/forge:latest`. The +management cluster must continue using `overlays/production`, whose resolver +annotation points at its internal `forge-core` ImageStream. + +Apply the Intlab-specific Application definition only in the Intlab cluster: + +```bash +oc apply -f gitops/applications/forge-intlab-production.yaml +``` + ## Deployment ### Option 1: Deploy Applications Directly @@ -110,6 +126,9 @@ oc apply -k gitops/overlays/development/ # Production oc apply -k gitops/overlays/production/ + +# Intlab production +oc apply -k gitops/overlays/intlab-production/ ``` ### Option 3: Manual Application Creation diff --git a/fournos/gitops/applications/forge-intlab-production.yaml b/fournos/gitops/applications/forge-intlab-production.yaml new file mode 100644 index 000000000..defdc2827 --- /dev/null +++ b/fournos/gitops/applications/forge-intlab-production.yaml @@ -0,0 +1,39 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + # The Application is still named forge-production because this is the + # production Forge instance in the Intlab cluster. This manifest is + # applied only to Intlab; management continues to use forge-production.yaml. + name: forge-production + namespace: openshift-gitops + labels: + app.kubernetes.io/name: forge-on-fournos + app.kubernetes.io/part-of: fournos + environment: intlab-production + annotations: + description: "FORGE Intlab production environment deployment via GitOps" +spec: + project: forge-on-fournos + source: + repoURL: https://github.com/openshift-psap/forge + targetRevision: main + path: fournos/gitops/overlays/intlab-production + destination: + server: https://kubernetes.default.svc + namespace: psap-automation + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=false + - PrunePropagationPolicy=foreground + - PruneLast=true + retry: + limit: 3 + backoff: + duration: 10s + factor: 2 + maxDuration: 5m + revisionHistoryLimit: 10 diff --git a/fournos/gitops/overlays/intlab-production/kustomization.yaml b/fournos/gitops/overlays/intlab-production/kustomization.yaml new file mode 100644 index 000000000..1a185c791 --- /dev/null +++ b/fournos/gitops/overlays/intlab-production/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: psap-automation + +resources: + - ../../base + +# Intlab does not run the OpenShift internal image registry. Keep resolver +# Jobs on the published Forge image; the management production overlay +# intentionally continues to use its internal forge-core ImageStream. +patches: + - target: + group: tekton.dev + version: v1 + kind: Pipeline + patch: |- + - op: replace + path: /metadata/annotations/fournos.dev~1resolve-image + value: "quay.io/rh_perfscale/forge:latest" + - target: + group: tekton.dev + version: v1 + kind: Task + name: forge-step + patch: |- + - op: replace + path: /spec/steps/0/image + value: "quay.io/rh_perfscale/forge:latest" + +commonLabels: + environment: intlab-production From f02591a3322bc5822a6b7e41ca60ee97482c4ff7 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 14:03:44 -0400 Subject: [PATCH 0886/1072] docs: clarify Intlab Forge deployment --- fournos/gitops/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fournos/gitops/README.md b/fournos/gitops/README.md index b71a73dc2..19405ed7c 100644 --- a/fournos/gitops/README.md +++ b/fournos/gitops/README.md @@ -115,6 +115,11 @@ oc apply -f gitops/applications/forge-intlab-production.yaml ## Deployment ### Option 1: Deploy Applications Directly + +This applies the default application set, including the management +`forge-production` definition. Do not use it for Intlab; use the +Intlab-specific Application definition below instead. + ```bash oc apply -k gitops/applications/ ``` From 1790c37df586c878fb39658bf7394f3b70290898 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 18:04:03 -0400 Subject: [PATCH 0887/1072] rhaiis: add B200 cluster preset --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index f43c75839..822707f5a 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -14,3 +14,7 @@ old-zeus: rhaiis.cluster_tag: "zeus1" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 + +b200: + rhaiis.cluster_tag: "b200" + rhaiis.gpu_types.nvidia: "b200" From 5bfb2bbf3d1b925fcee8bc1bfcde59bd10859b60 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 18:04:59 -0400 Subject: [PATCH 0888/1072] rhaiis: add MI355X cluster preset --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 822707f5a..591b2f79d 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -15,6 +15,9 @@ old-zeus: rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 +mi355x: + rhaiis.cluster_tag: "mi355x" + b200: rhaiis.cluster_tag: "b200" rhaiis.gpu_types.nvidia: "b200" From 8b33b5f2235ae51144d5ace515e827e104a40431 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 18:07:44 -0400 Subject: [PATCH 0889/1072] rhaiis: configure MI355X and B200 presets --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index 591b2f79d..be1635b3f 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -17,7 +17,11 @@ old-zeus: mi355x: rhaiis.cluster_tag: "mi355x" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 b200: rhaiis.cluster_tag: "b200" rhaiis.gpu_types.nvidia: "b200" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + benchmarks.guidellm.fs_group: 0 From 3c9440c3613f96137c66141364cc25d035ca3246 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 23:57:49 -0400 Subject: [PATCH 0890/1072] rhaiis: update default AMD vLLM image --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index bfc08f710..ea5530e85 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -13,7 +13,7 @@ engines: vllm: images: nvidia: vllm/vllm-openai:v0.24.0 - amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5 + amd: vllm/vllm-openai-rocm:v0.22.0 port: 8080 args: gpu-memory-utilization: 0.92 From 5c78f8613f9073d1e6bb08290dea332735c636f1 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 16 Sep 2026 23:59:11 -0400 Subject: [PATCH 0891/1072] rhaiis: use vLLM ROCm 0.26.0 --- projects/rhaiis/orchestration/config.d/rhaiis.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index ea5530e85..7b6acfddc 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -13,7 +13,7 @@ engines: vllm: images: nvidia: vllm/vllm-openai:v0.24.0 - amd: vllm/vllm-openai-rocm:v0.22.0 + amd: vllm/vllm-openai-rocm:v0.26.0 port: 8080 args: gpu-memory-utilization: 0.92 From 939a7891bd08063a911480ecd41ac9eed2c470b3 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 12:49:49 +0200 Subject: [PATCH 0892/1072] [core] notifications: integrate Jira notification destination --- projects/core/library/export.py | 4 +- projects/core/library/export_notifications.py | 53 +++++++------ projects/core/notifications/send.py | 75 ++++++++++++++++++- 3 files changed, 100 insertions(+), 32 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 92af75c28..382b45e20 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -31,7 +31,7 @@ TestPhase, _check_job_shutdown_status, _create_mlflow_url, - send_notification, + send_completion_notification, ) logger = logging.getLogger(__name__) @@ -399,7 +399,7 @@ def caliper_export_entrypoint( # Send completion notifications regardless of success/failure if status and not disable_notification: try: - notification_success = send_notification( + notification_success = send_completion_notification( artifact_dir, status, notification_provider=notification_provider, diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 7747b6cb4..6832fec07 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -23,7 +23,7 @@ from projects.core.library import config, env from projects.core.library.step_status import StepStatus from projects.core.notifications.provider import NotificationContext -from projects.core.notifications.send import send_notification as send_github_notification +from projects.core.notifications.send import send_notification as _send_github_notification logger = logging.getLogger(__name__) @@ -302,11 +302,12 @@ def _censor_notification_text(text: str, verbose: bool = False) -> str: raise -def send_notification( - artifact_dir: Path | None, +def send_completion_notification( + artifact_dir: Path, status: ExportStatus, notification_provider=None, dry_run: bool = False, + jira: bool = True, ) -> bool: """Send job completion notifications based on caliper export status. @@ -315,6 +316,7 @@ def send_notification( status: Caliper export status dataclass notification_provider: Optional per-project SlackNotificationProvider instance dry_run: If True, only build and log notification content without sending + jira: If True, send Jira notification (if configured) Returns: bool: True if notifications were sent successfully, False otherwise @@ -323,14 +325,14 @@ def send_notification( project = config.project.get_config("project.name") # Check individual step exit statuses from exit_status.yaml files once - step_status = None - if artifact_dir: - try: - step_status = _get_overall_step_status(artifact_dir) - if step_status == StepStatus.FAILURE: - logger.info("Step failure detected from exit_status.yaml files") - except Exception as e: - logger.warning(f"Failed to check step exit statuses for notification: {e}") + + try: + step_status = _get_overall_step_status(artifact_dir) + if step_status == StepStatus.FAILURE: + logger.info("Step failure detected from exit_status.yaml files") + except Exception as e: + logger.warning(f"Failed to check step exit statuses for notification: {e}") + step_status = None finish_reason = _extract_finish_reason_from_status(status, step_status) @@ -354,19 +356,13 @@ def send_notification( logger.info("Sending notification ...") # Write notification to file for GitHub pickup (always generate, even in dry-run) - try: - if env.ARTIFACT_DIR: - notification_file = Path(env.ARTIFACT_DIR) / "NOTIFICATION-github.md" - with open(notification_file, "w", encoding="utf-8") as f: - f.write(notification_status + "\n") - if dry_run: - logger.info(f"DRY RUN: Generated notification file {notification_file}") - else: - logger.info(f"Wrote export notification file {notification_file}") - else: - logger.warning("ARTIFACT_DIR not available, skipping notification file") - except Exception as e: - logger.exception(f"Failed to write notification file: {e}") + notification_file = env.ARTIFACT_DIR / "COMPLETION-NOTIFICATION.md" + notification_file.write_text(notification_status + "\n") + + if dry_run: + logger.info(f"DRY RUN: Generated notification file {notification_file}") + else: + logger.info(f"Wrote export notification file {notification_file}") # Actually send notification through GitHub API try: @@ -380,20 +376,21 @@ def send_notification( except Exception as e: logger.warning(f"Failed to get notification vault from config: {e}") - success = send_github_notification( + success = _send_github_notification( message=notification_status, github=True, slack=False, + jira=jira, dry_run=dry_run, notification_vault=notification_vault, ) if success: - logger.info("Successfully sent GitHub notification") + logger.info("Successfully sent GitHub and Jira notifications") else: - logger.error("GitHub notification sending failed") + logger.error("Notification sending failed") notification_success = False except Exception as e: - logger.error(f"Failed to send GitHub notification: {e}") + logger.error(f"Failed to send notifications: {e}") notification_success = False # Per-project Slack notification via provider diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 65bf31c57..964213b32 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -76,14 +76,21 @@ def get_secrets(notification_vault=None): def send_notification( - message, github=True, slack=False, dry_run=False, pr_number=None, notification_vault=None + message, + github=True, + slack=False, + jira=False, + dry_run=False, + pr_number=None, + notification_vault=None, ): - """Send a generic notification message to GitHub and/or Slack. + """Send a generic notification message to GitHub, Slack, and/or Jira. Args: message: The notification message content github: Whether to send to GitHub (default True) slack: Whether to send to Slack (default False) + jira: Whether to send to Jira (default False, requires notifications.jira config) dry_run: Whether to only log the message without sending (default False) pr_number: Optional PR number, auto-detected if None notification_vault: Optional vault name to get notification secrets from @@ -115,6 +122,10 @@ def send_notification( logger.error( "Cannot send Slack notification: no secrets available (vault or environment variables)" ) + if jira: + logger.error( + "Cannot send Jira notification: no secrets available (vault or environment variables)" + ) return False failed = False @@ -134,9 +145,69 @@ def send_notification( ): failed = True + if jira and not _send_notification_to_jira( + secret_dir, + message, + pr_number, + notification_vault, + dry_run, + ): + failed = True + return not failed +def _send_notification_to_jira(secret_dir, message, pr_number, notification_vault, dry_run): + """Send a notification to Jira if configured in the project config. + + Reads notifications.jira config to determine project_key and extra_tickets. + Creates or finds a ticket matching the PR number, then posts the message + as an ADF comment. + """ + try: + from projects.core.library import config + + jira_config = config.project.get_config("notifications.jira", {}) + except Exception: + jira_config = {} + + if not jira_config: + logger.info("No Jira notification config found, skipping") + return True + + try: + from projects.core.notifications.jira import api as jira_api + + jira_vault = jira_config.get("vault", notification_vault) + if jira_vault: + jira_secret_dir, _ = get_secrets(jira_vault) + if jira_secret_dir: + secret_dir = jira_secret_dir + + pr_title = jira_config.get("pr_title", "") + if not pr_title and pr_number: + try: + org, repo = get_org_repo() + _, pr_data = github_api.fetch_pr_data(org, repo, pr_number) + pr_title = pr_data.get("title", "") if pr_data else "" + except Exception as e: + logger.warning(f"Failed to fetch PR title from GitHub for Jira: {e}") + + return jira_api.send_jira_notification( + secret_dir=secret_dir, + project_key=jira_config["project_key"], + pr_number=pr_number or "NO_PR", + pr_title=pr_title, + markdown_content=message, + extra_tickets=jira_config.get("extra_tickets"), + dry_run=dry_run, + ) + + except Exception as e: + logger.error(f"Failed to send Jira notification: {e}") + return False + + ### From c984ed0f4c3d3191cefe65f42ff86c12b39a954d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:11:58 +0200 Subject: [PATCH 0893/1072] [core] ci_entrypoint: prepare_ci: remove jump-ci reference --- projects/core/ci_entrypoint/prepare_ci.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 01918677b..1b775ef9e 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -374,18 +374,6 @@ def system_prechecks() -> bool: f"File '{failures_file}' already exists, cannot continue. Set FORGE_IGNORE_FAILURES_FILE=1 to ignore this." ) - # Handle OpenShift CI PR arguments (already handled by parse_and_save_pr_arguments) - if ( - os.environ.get("OPENSHIFT_CI") == "true" - and os.environ.get("FORGE_JUMP_CI_INSIDE_JUMP_HOST") != "true" - ): - if not os.environ.get("FORGE_OPENSHIFT_CI_STEP_DIR"): - hostname = os.environ.get("HOSTNAME", "") - job_name_safe = os.environ.get("JOB_NAME_SAFE", "") - if hostname and job_name_safe: - step_dir = hostname.replace(f"{job_name_safe}-", "") + "/artifacts" - os.environ["FORGE_OPENSHIFT_CI_STEP_DIR"] = step_dir - # Remove any old failure markers old_failure = artifact_path / "FAILURE.txt" if old_failure.exists(): From 3594a6c2655636980b6bccc78db8b3350125bec8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:13:20 +0200 Subject: [PATCH 0894/1072] [fournos_launcher] orchestration: submit: isolate the OCPCI-specific code (link generation) --- projects/core/notifications/README.md | 6 +- .../fournos_launcher/orchestration/submit.py | 55 ++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/projects/core/notifications/README.md b/projects/core/notifications/README.md index e3bf9831b..6bd0c26d8 100644 --- a/projects/core/notifications/README.md +++ b/projects/core/notifications/README.md @@ -44,9 +44,8 @@ run my_project test ``` Any calling component -└── send_notification(message, github=True, slack=False) - ├── GitHub: send_notification_to_github() - └── Slack: send_notification_to_slack() +└── send_notification(message, github=True) + └── GitHub: send_notification_to_github() ``` ### Usage Example @@ -58,7 +57,6 @@ from projects.core.notifications.send import send_notification success = send_notification( message="🟢 Test completed successfully\nResults: https://example.com/results", github=True, - slack=False, dry_run=False, ) ``` diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index c93b325cb..19f554d61 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -1,3 +1,4 @@ +import json import logging import os import pathlib @@ -16,7 +17,7 @@ from projects.core.library import ci as ci_lib from projects.core.library import config, env, run, vault from projects.core.library.run_parallel import Parallel -from projects.core.notifications.send import get_ocpci_link, send_notification +from projects.core.notifications.send import send_notification from projects.fournos_launcher.orchestration import job_management, pr_args from projects.fournos_launcher.toolbox.cleanup_fjob.main import ( run as cleanup_fjob, @@ -68,6 +69,17 @@ def send_github_notification( error: Exception | None = None, ): """Send a simplified GitHub notification with essential test information.""" + + def get_ocpci_link(path, is_raw_file=False, base=None, is_dir=False): + if base is None: + base, suffix = get_ci_base_link(is_raw_file, is_dir) + else: + suffix = None + + link = base + (f"/{path}" if path else "") + (suffix if suffix else "") + + return link + try: artifact_dir = pathlib.Path(env.ARTIFACT_DIR) @@ -187,7 +199,7 @@ def send_github_notification( # Send notification through notification system try: - send_notification(message=notification_status, github=True, slack=False, dry_run=False) + send_notification(message=notification_status, github=True, dry_run=False) logger.info(f"Sent GitHub notification for {job_type} job") except Exception as send_error: logger.warning(f"Failed to send notification: {send_error}") @@ -196,6 +208,45 @@ def send_github_notification( logger.warning(f"Failed to send GitHub notification: {e}") +# returns a tuple (base_link, link_suffix) +def get_ci_base_link(is_raw_file=False, is_dir=False): + if os.environ.get("OPENSHIFT_CI") != "true": + logger.warning( + "Test not running from a well-known CI engine, cannot extract the artifacts link." + ) + + return "https://no_known_ci_engine/", "?no_ext=true" + + try: + job_spec = json.loads(os.environ["JOB_SPEC"]) + except KeyError: + logger.error("JOB_SPEC environment variable is not set") + raise + except json.JSONDecodeError as e: + logger.error("Failed to parse JOB_SPEC as JSON: %s", e) + raise + test_name = os.environ["JOB_NAME_SAFE"] + job = job_spec["job"] + build_id = job_spec["buildid"] + + if job_spec["type"] == "periodic": + link_path = f"logs/{job}/{build_id}" + + else: + pull_number = job_spec["refs"]["pulls"][0]["number"] + github_org = job_spec["refs"]["org"] + github_repo = job_spec["refs"]["repo"] + + link_path = f"pr-logs/pull/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}" + + return ( + "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/" + + link_path + + f"/artifacts/{test_name}", + "", + ) + + def init(): env.init() run.init() From 1bbd8acd445d68c8f222758eb9636c30a44622e6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:14:27 +0200 Subject: [PATCH 0895/1072] [core] notifications: remove slack/common and add Jira notifications --- projects/core/library/export.py | 6 +- projects/core/library/export_notifications.py | 223 +++--- projects/core/notifications/send.py | 709 ++---------------- .../fournos_launcher/orchestration/submit.py | 20 +- 4 files changed, 208 insertions(+), 750 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 382b45e20..8ca3df9ff 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -31,7 +31,7 @@ TestPhase, _check_job_shutdown_status, _create_mlflow_url, - send_completion_notification, + send_completion_notifications, ) logger = logging.getLogger(__name__) @@ -399,7 +399,7 @@ def caliper_export_entrypoint( # Send completion notifications regardless of success/failure if status and not disable_notification: try: - notification_success = send_completion_notification( + notification_success = send_completion_notifications( artifact_dir, status, notification_provider=notification_provider, @@ -534,7 +534,7 @@ def _update_artifacts( for fpath in [ artifact_dir_path / "run.log", - artifact_dir_path / "NOTIFICATION-github.md", + artifact_dir_path / "COMPLETION-NOTIFICATION.md", artifact_dir_path / "000__ci_metadata" / "fournos_fjob.yaml", ]: if fpath.is_file(): diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 6832fec07..b40278f75 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -23,10 +23,12 @@ from projects.core.library import config, env from projects.core.library.step_status import StepStatus from projects.core.notifications.provider import NotificationContext -from projects.core.notifications.send import send_notification as _send_github_notification +from projects.core.notifications.send import send_notification logger = logging.getLogger(__name__) +COMPLETION_NOTIFICATION_FILENAME = "COMPLETION-NOTIFICATION.md" + def _calculate_duration(start_time: str, end_time: str) -> str: """Calculate duration between two ISO timestamps and return formatted string.""" @@ -302,13 +304,115 @@ def _censor_notification_text(text: str, verbose: bool = False) -> str: raise -def send_completion_notification( +def _send_completion_message( + notification_status: str, + dry_run: bool, +) -> bool: + """Send the completion notification to GitHub and Jira via send_notification. + + Args: + notification_status: Censored markdown notification content + dry_run: If True, log but don't send + + Returns: + bool: True if all notifications succeeded + """ + try: + notification_vault = None + try: + notification_config = config.project.get_config("caliper.export.notifications", {}) + notification_vault = notification_config.get("vault") + if notification_vault: + logger.info(f"Using notification vault from config: {notification_vault}") + except Exception as e: + logger.warning(f"Failed to get notification vault from config: {e}") + + success = send_notification( + message=notification_status, + github=True, + jira=True, + dry_run=dry_run, + notification_vault=notification_vault, + ) + if success: + logger.info("Successfully sent completion notifications") + else: + logger.error("Completion notification sending failed") + + return success + + except Exception as e: + logger.error(f"Failed to send completion notifications: {e}") + return False + + +def _send_slack_provider_notification( + notification_provider, + status: ExportStatus, + finish_reason: str, + project: str, + dry_run: bool, +) -> bool: + """Send notification via the per-project Slack notification provider. + + Args: + notification_provider: SlackNotificationProvider instance + status: Export status dataclass (will be censored before sending) + finish_reason: Reason string for the notification context + project: Project name + dry_run: If True, log but don't send + + Returns: + bool: True if notification succeeded + """ + if not notification_provider: + logger.info("No slack notification provider, nothing to send.") + return True + + if dry_run: + logger.info( + f"DRY RUN: Would send per-project Slack notification with {notification_provider}" + ) + return True + + try: + artifact_dir = env.ARTIFACT_DIR + + try: + censored_status = yaml.safe_load( + _censor_notification_text(yaml.dump(status.to_dict()), verbose=False) + ) + except Exception as e: + logger.error(f"Slack notification censoring failed, aborting Slack notification: {e}") + return False + + context = NotificationContext( + status=censored_status, + finish_reason=str(finish_reason), + project_name=project or "unknown", + pr_number=os.environ.get("PULL_NUMBER"), + job_type=os.environ.get("JOB_TYPE"), + artifact_dir=artifact_dir, + ) + ok = notification_provider.notify(context) + if ok: + logger.info("Successfully sent per-project Slack notification") + else: + logger.warning("Per-project Slack notification failed") + + return ok + + except Exception as e: + logger.warning(f"Failed to send per-project Slack notification: {e}") + return False + + +def send_completion_notifications( artifact_dir: Path, status: ExportStatus, notification_provider=None, dry_run: bool = False, - jira: bool = True, -) -> bool: +): """Send job completion notifications based on caliper export status. Args: @@ -316,15 +420,12 @@ def send_completion_notification( status: Caliper export status dataclass notification_provider: Optional per-project SlackNotificationProvider instance dry_run: If True, only build and log notification content without sending - jira: If True, send Jira notification (if configured) Returns: bool: True if notifications were sent successfully, False otherwise """ - # Extract notification parameters from status object - project = config.project.get_config("project.name") - # Check individual step exit statuses from exit_status.yaml files once + project = config.project.get_config("project.name") try: step_status = _get_overall_step_status(artifact_dir) @@ -336,101 +437,53 @@ def send_completion_notification( finish_reason = _extract_finish_reason_from_status(status, step_status) - # Build enhanced notification with fournos job info and artifact links + success = True + success &= _send_detailed_completion_notification( + artifact_dir, + status, + finish_reason, + project, + step_status, + dry_run, + ) + + success &= _send_slack_provider_notification( + notification_provider, status, finish_reason, project, dry_run + ) + + return success + + +def _send_detailed_completion_notification( + artifact_dir: Path, + status: ExportStatus, + finish_reason: str, + project: str, + step_status, + dry_run: bool = False, +) -> bool: + notification_status, notification_success = _build_enhanced_notification( artifact_dir, project, finish_reason, status, step_status ) - # Apply censoring to notification content before sending try: notification_status = _censor_notification_text(notification_status, verbose=dry_run) except Exception as e: logger.error(f"Notification censoring failed, aborting notification delivery: {e}") return False - # Send actual notifications if dry_run: logger.info("DRY RUN: Would send notification") logger.info(f"DRY RUN: Notification content:\n{notification_status}") else: logger.info("Sending notification ...") - # Write notification to file for GitHub pickup (always generate, even in dry-run) - notification_file = env.ARTIFACT_DIR / "COMPLETION-NOTIFICATION.md" + notification_file = env.ARTIFACT_DIR / COMPLETION_NOTIFICATION_FILENAME notification_file.write_text(notification_status + "\n") + logger.info(f"Wrote export notification file {notification_file}") - if dry_run: - logger.info(f"DRY RUN: Generated notification file {notification_file}") - else: - logger.info(f"Wrote export notification file {notification_file}") - - # Actually send notification through GitHub API - try: - # Get notification vault from configuration - notification_vault = None - try: - notification_config = config.project.get_config("caliper.export.notifications", {}) - notification_vault = notification_config.get("vault") - if notification_vault: - logger.info(f"Using notification vault from config: {notification_vault}") - except Exception as e: - logger.warning(f"Failed to get notification vault from config: {e}") - - success = _send_github_notification( - message=notification_status, - github=True, - slack=False, - jira=jira, - dry_run=dry_run, - notification_vault=notification_vault, - ) - if success: - logger.info("Successfully sent GitHub and Jira notifications") - else: - logger.error("Notification sending failed") - notification_success = False - except Exception as e: - logger.error(f"Failed to send notifications: {e}") - notification_success = False - - # Per-project Slack notification via provider - if notification_provider: - if not dry_run: - try: - artifact_dir = Path(env.ARTIFACT_DIR) if env.ARTIFACT_DIR else None - # Censor status fields before creating NotificationContext - try: - censored_status = yaml.safe_load( - _censor_notification_text(yaml.dump(status.to_dict()), verbose=dry_run) - ) - except Exception as e: - logger.error( - f"Slack notification censoring failed, aborting Slack notification: {e}" - ) - notification_success = False - else: - # Only proceed with Slack notification if censoring succeeded - context = NotificationContext( - status=censored_status, - finish_reason=str(finish_reason), - project_name=project or "unknown", - pr_number=os.environ.get("PULL_NUMBER"), - job_type=os.environ.get("JOB_TYPE"), - artifact_dir=artifact_dir, - ) - ok = notification_provider.notify(context) - if ok: - logger.info("Successfully sent per-project Slack notification") - else: - logger.warning("Per-project Slack notification failed") - notification_success = False - except Exception as e: - logger.warning(f"Failed to send per-project Slack notification: {e}") - notification_success = False - else: - logger.info("DRY RUN: Would send per-project Slack notification") - - return notification_success + return notification_success and _send_completion_message(notification_status, dry_run) def _get_project_and_args(project: str, artifact_dir: Path | None) -> tuple[str, str, str]: diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index 964213b32..c248f9f85 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -1,14 +1,8 @@ -import html -import json import logging import os -import pathlib - -import yaml import projects.core.notifications.github.api as github_api -import projects.core.notifications.slack.api as slack_api -from projects.core.library import ci as ci_lib +from projects.core.library import vault as vault_lib logger = logging.getLogger(__name__) @@ -24,64 +18,11 @@ POSTPROCESS_STATUS_FILENAME = "postprocess_status.yaml" -def get_secrets(notification_vault=None): - """Get secrets directory, preferring vault system over environment variables. - - Args: - notification_vault: Name of the notification vault to use for secrets - - Returns: - tuple: (secret_dir_path, source_description) or (None, None) if not found - """ - # Try vault system first if vault name is provided - if notification_vault: - try: - from projects.core.library import vault as vault_lib - - # Get the vault manager and check if the vault exists - vault_manager = vault_lib.get_vault_manager() - vault_def = vault_manager._vault_cache.get(notification_vault) - if vault_def and vault_def.secret_dir and vault_def.secret_dir.exists(): - logger.info(f"Using notification secrets from vault: {notification_vault}") - return vault_def.secret_dir, f"vault:{notification_vault}" - else: - logger.warning(f"Vault {notification_vault} not found or directory doesn't exist") - except Exception as e: - logger.warning(f"Failed to get secrets from vault {notification_vault}: {e}") - - # Fall back to environment variables for backward compatibility - SECRET_ENV_KEYS = ( - "PSAP_FORGE_NOTIFICATIONS_SECRET_PATH", - "PSAP_FORGE_JUMP_CI_SECRET_PATH", - ) - - secret_env_key = None - warn = [] - for secret_env_key in SECRET_ENV_KEYS: - if os.environ.get(secret_env_key): - break - warn.append(f"{secret_env_key} not defined, cannot access the Github secrets") - else: - for warning in warn: - logger.warning(warning) - return None, None - - secret_dir = pathlib.Path(os.environ[secret_env_key]) - if not secret_dir.exists(): - logger.fatal(f"{secret_env_key} points to a non-existing directory ...") - return None, None - - logger.info(f"Using notification secrets from environment variable: {secret_env_key}") - return secret_dir, secret_env_key - - def send_notification( message, github=True, - slack=False, jira=False, dry_run=False, - pr_number=None, notification_vault=None, ): """Send a generic notification message to GitHub, Slack, and/or Jira. @@ -89,20 +30,13 @@ def send_notification( Args: message: The notification message content github: Whether to send to GitHub (default True) - slack: Whether to send to Slack (default False) jira: Whether to send to Jira (default False, requires notifications.jira config) dry_run: Whether to only log the message without sending (default False) - pr_number: Optional PR number, auto-detected if None notification_vault: Optional vault name to get notification secrets from Returns: bool: False if any notification failed, True if all succeeded """ - if pr_number is None: - pr_number = get_pr_number() - if not pr_number: - logger.warning("PR number not available, cannot send the GH notification") - github = False if not github_api: logger.info("Github API not available, don't send notification to github") @@ -112,44 +46,31 @@ def send_notification( logger.info("Running from a Periodic job, don't send notification to github") github = False - secret_dir, secret_env_key = get_secrets(notification_vault) - if secret_dir is None: + vault_def = vault_lib.get_vault_manager().get_vault(notification_vault) + if not vault_def: if github: logger.error( - "Cannot send GitHub notification: no secrets available (vault or environment variables)" - ) - if slack: - logger.error( - "Cannot send Slack notification: no secrets available (vault or environment variables)" + f"Cannot send GitHub notification: vault '{notification_vault}' not available" ) if jira: logger.error( - "Cannot send Jira notification: no secrets available (vault or environment variables)" + f"Cannot send Jira notification: vault '{notification_vault}' not available" ) - return False + + if not dry_run: + return False failed = False if github and not send_notification_to_github( - *get_github_secrets(secret_dir, secret_env_key), + vault_def, message, - pr_number, - dry_run, - ): - failed = True - - if slack and not send_notification_to_slack( - get_slack_secrets(secret_dir, secret_env_key), - message, - pr_number, dry_run, ): failed = True if jira and not _send_notification_to_jira( - secret_dir, + vault_def, message, - pr_number, - notification_vault, dry_run, ): failed = True @@ -157,7 +78,7 @@ def send_notification( return not failed -def _send_notification_to_jira(secret_dir, message, pr_number, notification_vault, dry_run): +def _send_notification_to_jira(vault_def, message, dry_run): """Send a notification to Jira if configured in the project config. Reads notifications.jira config to determine project_key and extra_tickets. @@ -178,26 +99,27 @@ def _send_notification_to_jira(secret_dir, message, pr_number, notification_vaul try: from projects.core.notifications.jira import api as jira_api - jira_vault = jira_config.get("vault", notification_vault) - if jira_vault: - jira_secret_dir, _ = get_secrets(jira_vault) - if jira_secret_dir: - secret_dir = jira_secret_dir - - pr_title = jira_config.get("pr_title", "") - if not pr_title and pr_number: - try: - org, repo = get_org_repo() - _, pr_data = github_api.fetch_pr_data(org, repo, pr_number) - pr_title = pr_data.get("title", "") if pr_data else "" - except Exception as e: - logger.warning(f"Failed to fetch PR title from GitHub for Jira: {e}") + server_file = vault_def.content.get("jira-server") + token_file = vault_def.content.get("jira-token") + + if not ( + server_file.file_path + and server_file.file_path.exists() + and token_file.file_path + and token_file.file_path.exists() + ): + logger.error( + f"jira-server or jira-token files not available in the '{vault_def.name}' vault" + ) + return False + + server = server_file.file_path.read_text().strip() + token = token_file.file_path.read_text().strip() return jira_api.send_jira_notification( - secret_dir=secret_dir, + server=server, + token=token, project_key=jira_config["project_key"], - pr_number=pr_number or "NO_PR", - pr_title=pr_title, markdown_content=message, extra_tickets=jira_config.get("extra_tickets"), dry_run=dry_run, @@ -211,12 +133,19 @@ def _send_notification_to_jira(secret_dir, message, pr_number, notification_vaul ### -def send_notification_to_github(pem_file, client_id, message, pr_number, dry_run): +def send_notification_to_github(vault_def, message, dry_run): """Send a generic notification message to GitHub.""" + + pem_file, client_id = get_github_secrets(vault_def) + pr_number = os.environ.get("PULL_NUMBER") org, repo = get_org_repo() abort = False + if None in (pem_file, client_id): + logger.error("github: Cannot access the Github notification secrets") + abort = True + if None in (pr_number,): logger.error("github: Cannot figure out the PR number") abort = True @@ -225,10 +154,6 @@ def send_notification_to_github(pem_file, client_id, message, pr_number, dry_run logger.error("github: Cannot access the org/repo") abort = True - if None in (pem_file, client_id): - logger.error("github: Cannot access the secret files") - abort = True - if abort: logger.error("github: Aborting due to previous error(s).") return False @@ -254,564 +179,30 @@ def send_notification_to_github(pem_file, client_id, message, pr_number, dry_run return resp.ok -def get_github_notification_message(finish_reason: str, status: str, pr_number: int): - def get_link(name, path, is_raw_file=False, base=None, is_dir=False): - return f"[{name}]({get_ocpci_link(path, is_raw_file, base, is_dir)})" - - def get_italics(text): - return f"*{text}*" - - def get_bold(text): - return f"**{text}**" - - status_icon = ":green_circle:" if finish_reason == "success" else ":red_circle:" - - return get_common_message( - finish_reason, - f"{status_icon} {status} {status_icon}", - get_link, - get_italics, - get_bold, - ) - - -def _get_notification_content(artifact_dir: pathlib.Path, get_link, get_bold) -> str: - """ - Extract notification content from 000__ci_metadata/notifications/ directory - - Args: - artifact_dir: Path to the artifact directory - get_link: Function to format links - get_bold: Function to format bold text - - Returns: - Formatted notification content string - """ - notifications_dir = ci_lib.get_ci_metadata_dir_location() / "notifications" - failures_file = artifact_dir / "FAILURES.txt" - - # Guard: Check if notifications directory exists - if not notifications_dir.exists(): - return _get_fallback_failure_content(failures_file, get_link, get_bold) - - # Guard: Check if there are any notification files - notification_files = sorted(notifications_dir.glob("*.txt")) - if not notification_files: - return _get_fallback_failure_content(failures_file, get_link, get_bold) - - content = "" - - # Process each notification file - for notification_file in notification_files: - try: - with open(notification_file, encoding="utf-8") as f: - notification_content = f.read().strip() - - # Guard: Skip empty files - if not notification_content: - continue - - # Extract title from filename (strip prefix until __ and .txt extension) - filename = notification_file.name - if "__" in filename: - title = filename.split("__", 1)[1].rsplit(".txt", 1)[0] - else: - title = filename.rsplit(".txt", 1)[0] - - # Replace underscores with spaces for better readability - title = title.replace("_", " ") - - content += f""" -{get_bold(title)}: -""" - # Format as quote blocks for better text wrapping - quoted_content = "\n".join( - f"> {line}" if line.strip() else ">" for line in notification_content.split("\n") - ) - content += f""" -{quoted_content} -""" - except Exception as e: - logger.warning(f"Failed to read {notification_file}: {e}") - content += f""" -• Error reading {notification_file.name} -""" - - # Add links to HTML reports - html_report_file = artifact_dir / "failure_analysis_report.html" - if html_report_file.exists(): - content += f""" -• {get_link("Detailed failure analysis report", "failure_analysis_report.html")} -""" - - config_review_file = artifact_dir / "config_review.html" - if config_review_file.exists(): - content += f""" -• {get_link("Config review report", "config_review.html")} -""" - - # Add link to FAILURES file if it exists (but don't include content) - if failures_file.exists(): - content += f""" -• {get_link("Raw failure details", "FAILURES.txt", is_raw_file=True)} -""" - - return content - - -def _get_fallback_failure_content(failures_file: pathlib.Path, get_link, get_bold) -> str: - """ - Fallback to legacy FAILURES file processing when no notifications exist - - Args: - failures_file: Path to the FAILURES file - get_link: Function to format links - get_bold: Function to format bold text - - Returns: - Formatted failure content string or empty string - """ - # Guard: Return early if FAILURES file doesn't exist - if not failures_file.exists(): - return "" - - try: - with open(failures_file) as f: - lines = f.readlines() - - # Guard: Return early for empty files - if not lines: - return """ -• Failure indicator: Empty. -""" - - DEFAULT_HEAD = 10 - try: - head = lines.index("---\n") - except ValueError: - head = DEFAULT_HEAD - - return f""" -• {get_link("Failure indicator", "FAILURES.txt", is_raw_file=True)}: -``` -{"".join(lines[:head])} -{"[...]" if len(lines) > head else ""} -``` -""" - except Exception as e: - logger.warning(f"Failed to read FAILURES file: {e}") - return f""" -• Error reading FAILURES file: {e} -""" - - -def get_common_message(finish_reason: str, status: str, get_link, get_italics, get_bold): - message = "" - - message += f"""\ -{get_bold(status)} -""" - - message += f""" -• Link to the {get_link("test results", "", is_dir=True)}. -""" - # Check for Caliper postprocess status and generated reports - artifact_dir = pathlib.Path(os.environ.get("ARTIFACT_DIR", "")) - caliper_status_path = None - - # Search for POSTPROCESS_STATUS_FILENAME in artifact directory and subdirectories - for status_file in artifact_dir.glob(f"**/{POSTPROCESS_STATUS_FILENAME}"): - caliper_status_path = status_file - break - - if caliper_status_path and caliper_status_path.exists(): - try: - with open(caliper_status_path) as f: - caliper_status = yaml.safe_load(f) - - visualize_step = caliper_status.get("steps", {}).get("visualize", {}) - if visualize_step.get("status") == "ok" and visualize_step.get("paths"): - paths = visualize_step["paths"] - message += f""" -• Generated {len(paths)} Caliper report(s): -""" - for path in paths: - # Just list the path, no link - message += f" - {path}\n" - - # Also link to the reports index if it exists - reports_index_path = artifact_dir / "reports_index.html" - if reports_index_path.exists(): - message += f""" -• Link to the {get_link("reports index", "reports_index.html")}. -""" - else: - message += """ -• Caliper postprocess completed but no reports generated. -""" - except Exception as e: - logger.warning("Failed to parse POSTPROCESS_STATUS_FILENAME: %s", e) - message += """ -• Failed to parse POSTPROCESS_STATUS_FILENAME ... -""" - - # Include fournos_launcher generated notification content - fournos_notification_md = artifact_dir / "NOTIFICATION-github.md" - if fournos_notification_md.exists(): - try: - with open(fournos_notification_md, encoding="utf-8") as f: - fournos_content = f.read().strip() - if fournos_content: - message += f""" -{fournos_content} -""" - except Exception as e: - logger.warning("Failed to read NOTIFICATION-github.md: %s", e) - - if (var_over := ci_lib.get_ci_metadata_dir_location() / "pr_config.txt").exists(): - with open(var_over) as f: - message += f""" -{get_bold("Test configuration")}: -``` -{f.read().strip()} -``` -""" - elif (var_over := ci_lib.get_ci_metadata_dir_location() / "variable_overrides.yaml").exists(): - with open(var_over) as f: - message += f""" -{get_bold("Test configuration")}: -``` -{f.read().strip()} -``` -""" - else: - message += """ -• No test configuration (`variable_overrides.yaml/pr_config.txt`) available. -""" - - # Get notification content from dedicated function - artifact_dir = pathlib.Path(os.environ.get("ARTIFACT_DIR", "")) - notification_content = _get_notification_content(artifact_dir, get_link, get_bold) - message += notification_content - - message += "• " + get_link("Execution logs", "run.log", is_raw_file=True) - - return message - - -# Warning: -# Slack API messages format is different from the GUI -# https://api.slack.com/reference/surfaces/formatting - - -def get_slack_thread_message(finish_reason, status): - def get_link(name, path, is_raw_file=False, base=None, is_dir=False): - return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" - - def get_italics(text): - return f"_{text}_" - - def get_bold(text): - return f"*{text}*" - - status_icon = ":done-circle-check:" if finish_reason == "success" else ":no-red-circle:" - - # Check for fournos_launcher generated Slack notification content - artifact_dir = pathlib.Path(os.environ.get("ARTIFACT_DIR", "")) - fournos_notification_md = artifact_dir / "NOTIFICATION.md" - - if fournos_notification_md.exists(): - try: - with open(fournos_notification_md, encoding="utf-8") as f: - fournos_content = f.read().strip() - if fournos_content: - # Return custom notification content with status icon - return f"{status_icon} {get_bold(status)}\n\n{fournos_content}" - except (OSError, UnicodeDecodeError) as e: - logger.warning(f"Failed to read {fournos_notification_md}: {e}") - # Fall through to default message - - # Fallback to standard message if no custom notification - return get_common_message( - finish_reason, f"{status_icon} {status}", get_link, get_italics, get_bold - ) - - -def get_slack_channel_message(anchor: str, pr_data: dict): - """Generates the Slack's notification main thread message.""" - - org, repo = get_org_repo() - - message = f"🧵 {anchor}" - - if not pr_data: - return message - - message += f""" - -```{pr_data["title"]}``` - -Link to the <{pr_data["html_url"]}|PR>. -""" - - return message - - -def send_notification_to_slack( - token, - message, - pr_number, - dry_run, -): - """Send a generic notification message to Slack.""" - if not token: - return False - - client = slack_api.init_client(token) - if not client: - return False - - org, repo = get_org_repo() - is_periodic = False - pr_data = None - pr_created_at = None - - if pr_number: - if github_api: - pr_created_at, pr_data = github_api.fetch_pr_data(org, repo, pr_number) - - anchor = f"Thread for PR #{pr_number}" - elif os.environ.get("JOB_TYPE") == "periodic": - periodic_name = os.environ["JOB_NAME_SAFE"] - anchor = f"Thread for Periodic job `{periodic_name}`" - is_periodic = True - else: - anchor = "Thread for tests without PRs" - - channel_msg_ts, channel_message = slack_api.search_channel_message( - client, anchor, not_before=pr_created_at +def get_org_repo(): + return ( + os.environ.get("REPO_OWNER", DEFAULT_REPO_OWNER), + os.environ.get("REPO_NAME", DEFAULT_REPO_NAME), ) - if not channel_msg_ts: - if is_periodic: - channel_message = anchor - else: - channel_message = get_slack_channel_message(anchor, pr_data) - - if dry_run: - logger.info("Posting Slack channel notification ...") - else: - channel_msg_ts, ok = slack_api.send_message(client, message=channel_message) - if not ok: - return False - - if dry_run: - logger.info(f"Slack channel notification:\n{channel_message}") - logger.info(f"Slack thread notification:\n{message}") - logger.info("***") - logger.info("***") - logger.info("***\n") - - return True - - _, ok = slack_api.send_message(client, message=message, main_ts=channel_msg_ts) - return ok +def get_github_secrets(vault_def): + pem_file = vault_def.content.get(GITHUB_APP_PEM_FILE).file_path + client_id_file = vault_def.content.get(GITHUB_APP_CLIENT_ID_FILE).file_path - -### - - -def get_pr_number(): - return os.environ.get("PULL_NUMBER") - - -# returns a tuple (base_link, link_suffix) -def get_ci_base_link(is_raw_file=False, is_dir=False): - if os.environ.get("OPENSHIFT_CI") == "true": - try: - job_spec = json.loads(os.environ["JOB_SPEC"]) - except KeyError: - logger.error("JOB_SPEC environment variable is not set") - raise - except json.JSONDecodeError as e: - logger.error("Failed to parse JOB_SPEC as JSON: %s", e) - raise - test_name = os.environ["JOB_NAME_SAFE"] - test_path = os.environ.get( - "FORGE_OPENSHIFT_CI_STEP_DIR", "FORGE_OPENSHIFT_CI_STEP_DIR_missing" - ) - job = job_spec["job"] - build_id = job_spec["buildid"] - - if job_spec["type"] == "periodic": - link_path = f"logs/{job}/{build_id}" - - else: - pull_number = job_spec["refs"]["pulls"][0]["number"] - github_org = job_spec["refs"]["org"] - github_repo = job_spec["refs"]["repo"] - - link_path = f"pr-logs/pull/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}" - - return ( - "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results-public/" - + link_path - + f"/artifacts/{test_name}/{test_path}", - "", - ) - - else: + if not pem_file: logger.warning( - "Test not running from a well-known CI engine, cannot extract the artifacts link." + f"Github App private key does not exists ({GITHUB_APP_PEM_FILE}) in {vault_def.name}" ) - return "https://no_known_ci_engine/", "?no_ext=true" - - -def get_org_repo(): - return os.environ.get("REPO_OWNER", DEFAULT_REPO_OWNER), os.environ.get( - "REPO_NAME", DEFAULT_REPO_NAME - ) - - -def get_github_secrets(secret_dir, secret_env_key): - pem_file = secret_dir / GITHUB_APP_PEM_FILE - client_id_file = secret_dir / GITHUB_APP_CLIENT_ID_FILE - - if not pem_file.exists(): - logger.warning(f"Github App private key does not exists ({pem_file}) in {secret_env_key}") - return None, None - - if not client_id_file.exists(): + if not client_id_file: logger.warning( - f"Github App clientid file does not exists ({client_id_file}) in {secret_env_key}" + f"Github App clientid file does not exists ({GITHUB_APP_CLIENT_ID_FILE}) in {vault_def.name}" ) + + if not (pem_file and client_id_file): return None, None client_id_content = client_id_file.read_text().strip() return pem_file, client_id_content - - -def get_slack_secrets(secret_dir, secret_env_key): - token_file = secret_dir / SLACK_TOKEN_FILE - - if not token_file.exists(): - logger.warning( - f"{token_file.name} not found in {secret_env_key}. Cannot send the Slack notification" - ) - return None - - return token_file.read_text() - - -def get_ocpci_link(path, is_raw_file=False, base=None, is_dir=False): - if base is None: - base, suffix = get_ci_base_link(is_raw_file, is_dir) - else: - suffix = None - - link = base + (f"/{path}" if path else "") + (suffix if suffix else "") - - return link - - -""" # example of a regression_summary file: -entries_count: 3 -failures: 0 -kpis_count: 2 -message: Performed 6 KPI regression analyses over 3 entries x 2 KPIs. 0 KPIs didn't - pass. -no_history: 0 -not_analyzed: 0 -significant_performance_increase: 0 -total_points: 6 -""" - - -def send_cpt_notification(regression_summary_path, title, slack, dry_run): - summary_path = pathlib.Path(regression_summary_path) - if not summary_path.exists(): - logger.fatal(f"Regression summary doesn't exist :/ ({regression_summary_path})") - return True - - try: - with open(summary_path) as f: - summary = yaml.safe_load(f) - except Exception as e: - logger.fatal(f"Failed to load regression summary: {e}") - return True - - secret_dir, secret_env_key = get_secrets(None) - if secret_dir is None: - return True - - failed = False - if slack: - failed = send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, dry_run) - - return failed - - -def send_cpt_notification_to_slack(secret_dir, secret_env_key, title, summary, dry_run): - token = get_slack_secrets(secret_dir, secret_env_key) - if not token: - return True - - client = slack_api.init_client(token) - if not client: - logger.fatal("Couldn't get the slack client ...") - return True - safe_title = html.escape(title) - channel_msg_ts, channel_message = slack_api.search_channel_message(client, safe_title) - - if not channel_msg_ts: - channel_message = f"🧵 Thread for `{title}` continuous performance testing" - if dry_run: - logger.info(f"Posting Slack channel notification ...\n{channel_message}") - else: - channel_msg_ts, channel_ok = slack_api.send_message(client, message=channel_message) - - try: - thread_message = get_slack_cpt_message(summary) - except Exception as e: - logger.fatal(f"Failed to generate the slack notification message: {e}") - return True - - if dry_run: - logger.info(f"Posting Slack thread notification ...\n{thread_message}") - ok = True - else: - _, ok = slack_api.send_message(client, message=thread_message, main_ts=channel_msg_ts) - - return not ok - - -def get_slack_cpt_message(summary): - def get_link(name, path, is_raw_file=False, base=None, is_dir=False): - return f"<{get_ocpci_link(path, is_raw_file, base, is_dir)}|{name}>" - - def get_italics(text): - return f"_{text}_" - - def get_bold(text): - return f"*{text}*" - - status_icon = ":no-red-circle:" if summary.get("failures") else ":done-circle-check:" - - reports_index_link = "" - if (pathlib.Path(os.environ.get("ARTIFACT_DIR", "")) / "reports_index.html").exists(): - reports_index_link = f"• Link to the {get_link('reports index', 'reports_index.html')}.\n" - - return f"""{status_icon} {get_bold(summary["message"])} - -• Link to the {get_link("test results", "", is_dir=True)}. -{reports_index_link} -- `{summary["entries_count"]}` entries were tested against `{summary["kpis_count"]}` KPIs -- `{summary["failures"]}` failed -- `{summary["no_history"]}` had no history -- `{summary["not_analyzed"]}` were not analyzed -- `{summary["significant_performance_increase"]}` had a significant performance degradation -- `{summary["total_points"]}` points were checked for regression. - """ diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index 19f554d61..b58cb83e3 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -189,7 +189,7 @@ def get_ocpci_link(path, is_raw_file=False, base=None, is_dir=False): # Write simplified notification to file for notification system pickup try: - notification_file = artifact_dir / "NOTIFICATION-github.md" + notification_file = artifact_dir / "COMPLETION-NOTIFICATION.md" with open(notification_file, "w", encoding="utf-8") as f: f.write(notification_status) logger.info(f"Created notification content for {job_type} job") @@ -199,8 +199,22 @@ def get_ocpci_link(path, is_raw_file=False, base=None, is_dir=False): # Send notification through notification system try: - send_notification(message=notification_status, github=True, dry_run=False) - logger.info(f"Sent GitHub notification for {job_type} job") + notification_vault = None + try: + notification_vault = config.project.get_config("caliper.export.notifications.vault") + except Exception: + pass + + ok = send_notification( + message=notification_status, + github=True, + dry_run=False, + notification_vault=notification_vault, + ) + if ok: + logger.info(f"Sent GitHub notification for {job_type} job") + else: + logger.warning(f"GitHub notification for {job_type} job returned failure") except Exception as send_error: logger.warning(f"Failed to send notification: {send_error}") From 77137cc2b80db7ce8c8da52a235f0abf39d23716 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:14:46 +0200 Subject: [PATCH 0896/1072] vaults: psap-forge-notifications: add jira secrets to the vault --- vaults/psap-forge-notifications.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 73eb1ce5b..268857b06 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -14,3 +14,11 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot + + jira-server: + description: | + Jira server URL (e.g. https://issues.redhat.com) + + jira-token: + description: | + Jira Personal Access Token for authentication From c35c9ddab83cab95b03e3ac4cdbd46a1c30d68ec Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:42:23 +0200 Subject: [PATCH 0897/1072] [core] library: vault: add the ability to auto-discover vaults from PSAP_VAULT_DIRECTORY --- docs/orchestration/vaults.md | 54 ++++++++++++++++++++++++++--- projects/core/library/vault.py | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/docs/orchestration/vaults.md b/docs/orchestration/vaults.md index 44a8928ce..d6d60bb7a 100644 --- a/docs/orchestration/vaults.md +++ b/docs/orchestration/vaults.md @@ -166,15 +166,51 @@ kubeconfig_path = vault.content["kubeconfig"].file_path ## Environment Setup -### Required Environment Variables +### Auto-Discovery with `FORGE_VAULT_DIRECTORY` -Each vault requires its corresponding environment variable to be set: +Instead of setting individual environment variables for each vault, you +can point `FORGE_VAULT_DIRECTORY` to a single directory containing all +your vault subdirectories: + +```bash +export FORGE_VAULT_DIRECTORY="/path/to/creds" +``` + +The directory should contain subdirectories named after vault +definitions (matching the YAML filenames in `$FORGE_HOME/vaults/`): + +``` +/path/to/creds/ +├── psap-forge-ci/ +│ ├── kubeconfig +│ └── server_url +├── psap-forge-hf/ +│ └── hf_token +├── psap-forge-mlflow-export/ +│ └── mlflow-secret.yaml +└── ... +``` + +On `vault.init()`, the vault module scans this directory and +automatically sets the corresponding environment variables (e.g., +`PSAP_FORGE_FOURNOS_CI_SECRET_PATH=/path/to/creds/psap-forge-ci`). +Subdirectories that don't match any vault definition are silently +skipped. Environment variables that are already set are not overridden. + +This mechanism is only active outside of Fournos CI (where +`FOURNOS_SECRETS` handles vault setup instead). + +### Individual Environment Variables + +Alternatively, each vault can be configured with its own environment variable: ```bash export PSAP_FORGE_FOURNOS_CI_SECRET_PATH="/path/to/ci/secrets" export PSAP_FORGE_NOTIFICATIONS_SECRET_PATH="/path/to/notification/secrets" ``` +These take precedence over `FORGE_VAULT_DIRECTORY` auto-discovery. + ### Directory Structure Vault directories should contain only the files defined in the vault specification: @@ -212,6 +248,14 @@ content: ### 2. Set Environment Variable +If using `FORGE_VAULT_DIRECTORY`, create the vault subdirectory under it: + +```bash +mkdir -p $FORGE_VAULT_DIRECTORY/my-new-vault +``` + +Otherwise, set the individual environment variable: + ```bash export MY_VAULT_SECRET_PATH="/path/to/my/secrets" ``` @@ -219,9 +263,9 @@ export MY_VAULT_SECRET_PATH="/path/to/my/secrets" ### 3. Create Secret Files ```bash -mkdir -p /path/to/my/secrets -echo "secret-key-value" > /path/to/my/secrets/api_key -cp service-credentials.json /path/to/my/secrets/credentials.json +# Use $FORGE_VAULT_DIRECTORY/my-new-vault or $MY_VAULT_SECRET_PATH +echo "secret-key-value" > $FORGE_VAULT_DIRECTORY/my-new-vault/api_key +cp service-credentials.json $FORGE_VAULT_DIRECTORY/my-new-vault/credentials.json ``` ### 4. Add to Project Requirements diff --git a/projects/core/library/vault.py b/projects/core/library/vault.py index 262da77f5..c4d878ef2 100644 --- a/projects/core/library/vault.py +++ b/projects/core/library/vault.py @@ -423,6 +423,64 @@ def is_strict_validation_enabled() -> bool: return _strict_validation_enabled +def init_from_directory(base_dir: Path) -> list[str]: + """Auto-discover vaults from a base directory. + + Scans base_dir for subdirectories matching vault definition names + (from vaults/*.yaml) and sets the corresponding env_key environment + variables so that the vault system can locate the secret files. + + Does not override environment variables that are already set. + + Args: + base_dir: Directory containing vault subdirectories + + Returns: + List of discovered vault names + """ + vaults_def_dir = env.FORGE_HOME / "vaults" + if not vaults_def_dir.exists(): + logger.warning(f"Vault definitions directory does not exist: {vaults_def_dir}") + return [] + + if not base_dir.is_dir(): + logger.error(f"FORGE_VAULT_DIRECTORY is not a directory: {base_dir}") + return [] + + discovered = [] + for subdir in sorted(base_dir.iterdir()): + if not subdir.is_dir(): + continue + + vault_def_file = vaults_def_dir / f"{subdir.name}.yaml" + if not vault_def_file.exists(): + logger.debug(f"No vault definition for directory: {subdir.name}") + continue + + try: + with open(vault_def_file) as f: + vault_def = yaml.safe_load(f) + except Exception as e: + logger.error(f"Failed to load vault definition {vault_def_file}: {e}") + continue + + env_key = vault_def.get("env_key") + if not env_key: + logger.error(f"Missing env_key in vault definition: {vault_def_file}") + continue + + if env_key in os.environ: + logger.debug(f"Vault '{subdir.name}': {env_key} already set, not overriding") + else: + os.environ[env_key] = str(subdir) + logger.info(f"Vault '{subdir.name}': set {env_key}={subdir}") + + discovered.append(subdir.name) + + logger.info(f"Auto-discovered {len(discovered)} vaults from {base_dir}: {discovered}") + return discovered + + def init( vaults: list[str] = None, mandatory_vaults: list[str] = None, @@ -448,6 +506,11 @@ def init( logger.warning("VaultManager already initialized", stack_info=True) return + if not env.running_inside_fournos(): + base_dir = os.environ.get("FORGE_VAULT_DIRECTORY") + if base_dir: + init_from_directory(Path(base_dir)) + _vault_manager = VaultManager() # Handle legacy single vaults list (treated as mandatory) From ea4651a35aefbe85fb4429de5bfb2906c283c4fe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 14:53:08 +0200 Subject: [PATCH 0898/1072] [core] notifications: jira: new API --- projects/core/notifications/jira/__init__.py | 0 projects/core/notifications/jira/api.py | 160 +++++++++++++++++++ projects/core/notifications/send.py | 23 ++- pyproject.toml | 2 + 4 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 projects/core/notifications/jira/__init__.py create mode 100644 projects/core/notifications/jira/api.py diff --git a/projects/core/notifications/jira/__init__.py b/projects/core/notifications/jira/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/projects/core/notifications/jira/api.py b/projects/core/notifications/jira/api.py new file mode 100644 index 000000000..306265ed4 --- /dev/null +++ b/projects/core/notifications/jira/api.py @@ -0,0 +1,160 @@ +import logging + +logger = logging.getLogger(__name__) + + +def get_client(server, token): + """Create a Jira client using a Personal Access Token. + + Args: + server: Jira server URL (e.g. https://issues.redhat.com) + token: Personal Access Token + + Returns: + Jira client instance + """ + from atlassian import Jira + + return Jira(url=server, token=token) + + +def find_or_create_ticket(client, project_key, pr_number, pr_title): + """Find an existing ticket for this PR or create a new one. + + Args: + client: Jira client + project_key: Jira project key (e.g. PSAP-CI) + pr_number: GitHub PR number + pr_title: GitHub PR title (used when creating) + + Returns: + str: Jira ticket key (e.g. PSAP-CI-42) + """ + jql = f'project = "{project_key}" AND summary ~ "PR #{pr_number}"' + logger.info(f"Searching Jira with JQL: {jql}") + + results = client.post( + "rest/api/3/search/jql", + data={"jql": jql, "fields": ["key", "summary"]}, + ) + issues = results.get("issues", []) + + if issues: + ticket_key = issues[0]["key"] + logger.info(f"Found existing Jira ticket: {ticket_key}") + return ticket_key + + summary = f"[Forge] PR #{pr_number}: {pr_title}" + logger.info(f"Creating new Jira ticket in {project_key}: {summary}") + + result = client.issue_create( + fields={ + "project": {"key": project_key}, + "summary": summary, + "issuetype": {"name": "Task"}, + } + ) + + ticket_key = result["key"] + logger.info(f"Created Jira ticket: {ticket_key}") + + return ticket_key + + +def markdown_to_adf(markdown_text): + """Convert GitHub-flavored Markdown to Atlassian Document Format. + + Args: + markdown_text: Markdown string + + Returns: + dict: ADF document ({"version": 1, "type": "doc", "content": [...]}) + """ + from marklassian import markdown_to_adf as _markdown_to_adf + + return _markdown_to_adf(markdown_text) + + +def post_comment(client, ticket_key, markdown_content, dry_run=False): + """Post a comment in ADF format to a Jira ticket. + + Args: + client: Jira client + ticket_key: Jira ticket key (e.g. PSAP-CI-42) + markdown_content: Markdown string to convert and post + dry_run: If True, log but don't post + + Returns: + bool: True if successful + """ + adf_body = markdown_to_adf(markdown_content) + + if dry_run: + logger.info(f"DRY RUN: Would post ADF comment to {ticket_key}") + return True + + logger.info(f"Posting ADF comment to {ticket_key}") + response = client.post( + f"rest/api/3/issue/{ticket_key}/comment", + data={"body": adf_body}, + ) + + if response and not isinstance(response, dict): + logger.error(f"Failed to post comment to {ticket_key}: {response}") + return False + + logger.info(f"Successfully posted comment to {ticket_key}") + return True + + +def send_jira_notification( + server, + token, + project_key, + pr_number, + pr_title, + markdown_content, + extra_tickets=None, + dry_run=False, +): + """Send a notification to Jira: find/create a ticket and post a comment. + + Args: + server: the Jira server address + token: the client token to use + project_key: Jira project key (e.g. PSAP-CI) + pr_number: GitHub PR number + pr_title: GitHub PR title + markdown_content: Markdown notification content + extra_tickets: Optional str or list of additional ticket keys to post to + dry_run: If True, log but don't post + + Returns: + bool: True if all notifications succeeded + """ + + if not server or not token: + raise ValueError("Server or token missing") + + client = get_client(server, token) + success = True + + ticket_key = find_or_create_ticket(client, project_key, pr_number, pr_title) + if not ticket_key: + logger.error("Failed to find or create Jira ticket") + return False + + if not post_comment(client, ticket_key, markdown_content, dry_run=dry_run): + success = False + + if extra_tickets: + if isinstance(extra_tickets, str): + extra_tickets = [extra_tickets] + + for extra_key in extra_tickets: + logger.info(f"Posting to extra ticket: {extra_key}") + if not post_comment(client, extra_key, markdown_content, dry_run=dry_run): + logger.warning(f"Failed to post comment to extra ticket {extra_key}") + success = False + + return success diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index c248f9f85..c9bf2e2d8 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -85,17 +85,6 @@ def _send_notification_to_jira(vault_def, message, dry_run): Creates or finds a ticket matching the PR number, then posts the message as an ADF comment. """ - try: - from projects.core.library import config - - jira_config = config.project.get_config("notifications.jira", {}) - except Exception: - jira_config = {} - - if not jira_config: - logger.info("No Jira notification config found, skipping") - return True - try: from projects.core.notifications.jira import api as jira_api @@ -116,12 +105,20 @@ def _send_notification_to_jira(vault_def, message, dry_run): server = server_file.file_path.read_text().strip() token = token_file.file_path.read_text().strip() + # should be retrieved from the configuration + pr_number = "1234" + pr_title = "dummy project" + project_key = "PSAPCI" + extra_tickets = [] + return jira_api.send_jira_notification( server=server, token=token, - project_key=jira_config["project_key"], + pr_number=pr_number, + pr_title=pr_title, + project_key=project_key, markdown_content=message, - extra_tickets=jira_config.get("extra_tickets"), + extra_tickets=extra_tickets, dry_run=dry_run, ) diff --git a/pyproject.toml b/pyproject.toml index 51f7237de..16af58628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,8 @@ dependencies = [ "jsonpath_ng", "pydantic>=2.0", "yq", + "marklassian", + "atlassian-python-api", ] [project.optional-dependencies] From df627747b521a00c6c0d74f3c942f0da8a3576eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 16:19:16 +0200 Subject: [PATCH 0899/1072] [caliper] orchestration: censoring: simplify AI code ... --- projects/caliper/orchestration/censoring.py | 80 ++++++--------------- 1 file changed, 21 insertions(+), 59 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index e05eab431..7009940ac 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -62,67 +62,29 @@ def discover_vault_secrets( ) continue - # If censor_text is available, use it directly instead of reading file - if content_def.censor_text: - content_path = vault_manager.get_vault_content_path(vault_name, content_name) - if not (content_path and content_path.exists()): - logger.warning( - f"Invalid vault found: {vault_name} {content_name} (missing)" - ) - continue - - try: - # Read actual vault content - content_text = content_path.read_text( - encoding="utf-8", errors="ignore" - ).strip() - if not content_text: - logger.warning( - f"Invalid vault found: {vault_name} {content_name} (empty)" - ) - else: - vault_secrets.add(content_text) - secret_mapping[content_text] = f"{vault_name}/{content_name}" + content_path = vault_manager.get_vault_content_path(vault_name, content_name) + if not (content_path and content_path.exists()): + logger.warning(f"Invalid vault found: {vault_name} {content_name} (missing)") + continue + + try: + # Read actual vault content + content_text = content_path.read_text(encoding="utf-8", errors="ignore").strip() + if not content_text: + logger.warning(f"Invalid vault found: {vault_name} {content_name} (empty)") + else: + vault_secrets.add(content_text) + secret_mapping[content_text] = f"{vault_name}/{content_name}" + if content_def.censor_text: censor_text_mapping[content_text] = content_def.censor_text - secrets_discovered += 1 - if verbose: - logger.info( - f"Discovered secret from vault {vault_name}/{content_name} with censor_text" - ) - except Exception as e: - logger.warning( - f"Failed to read vault content {vault_name}/{content_name}: {e}" - ) - else: - # No censor_text, read file content for traditional censoring - content_path = vault_manager.get_vault_content_path(vault_name, content_name) - if not (content_path and content_path.exists()): - logger.warning( - f"Invalid vault found: {vault_name} {content_name} (missing)" - ) - continue - - try: - # Read vault content (assume it's text) - content_text = content_path.read_text( - encoding="utf-8", errors="ignore" - ).strip() - if not content_text: - logger.warning( - f"Invalid vault found: {vault_name} {content_name} (empty)" + secrets_discovered += 1 + + if verbose: + logger.info( + f"Discovered secret from vault {vault_name}/{content_name}{' with censor_text' if content_def.censor_text else ''}" ) - else: - vault_secrets.add(content_text) - secret_mapping[content_text] = f"{vault_name}/{content_name}" - secrets_discovered += 1 - if verbose: - logger.info( - f"Discovered secret from vault {vault_name}/{content_name}" - ) - except Exception as e: - logger.warning( - f"Failed to read vault content {vault_name}/{content_name}: {e}" - ) + except Exception as e: + logger.warning(f"Failed to read vault content {vault_name}/{content_name}: {e}") if verbose: logger.info(f"Discovered {secrets_discovered} vault secrets for censoring") From 97257a9b4e26f6cdad5e3e14aca8ec15361433b0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 16:38:24 +0200 Subject: [PATCH 0900/1072] [core] Remove the JIRA integration stub --- projects/core/library/export_notifications.py | 1 - projects/core/notifications/jira/__init__.py | 0 projects/core/notifications/jira/api.py | 160 ------------------ projects/core/notifications/send.py | 65 ------- pyproject.toml | 2 - vaults/psap-forge-notifications.yaml | 8 - 6 files changed, 236 deletions(-) delete mode 100644 projects/core/notifications/jira/__init__.py delete mode 100644 projects/core/notifications/jira/api.py diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index b40278f75..ed2af7850 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -330,7 +330,6 @@ def _send_completion_message( success = send_notification( message=notification_status, github=True, - jira=True, dry_run=dry_run, notification_vault=notification_vault, ) diff --git a/projects/core/notifications/jira/__init__.py b/projects/core/notifications/jira/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/projects/core/notifications/jira/api.py b/projects/core/notifications/jira/api.py deleted file mode 100644 index 306265ed4..000000000 --- a/projects/core/notifications/jira/api.py +++ /dev/null @@ -1,160 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) - - -def get_client(server, token): - """Create a Jira client using a Personal Access Token. - - Args: - server: Jira server URL (e.g. https://issues.redhat.com) - token: Personal Access Token - - Returns: - Jira client instance - """ - from atlassian import Jira - - return Jira(url=server, token=token) - - -def find_or_create_ticket(client, project_key, pr_number, pr_title): - """Find an existing ticket for this PR or create a new one. - - Args: - client: Jira client - project_key: Jira project key (e.g. PSAP-CI) - pr_number: GitHub PR number - pr_title: GitHub PR title (used when creating) - - Returns: - str: Jira ticket key (e.g. PSAP-CI-42) - """ - jql = f'project = "{project_key}" AND summary ~ "PR #{pr_number}"' - logger.info(f"Searching Jira with JQL: {jql}") - - results = client.post( - "rest/api/3/search/jql", - data={"jql": jql, "fields": ["key", "summary"]}, - ) - issues = results.get("issues", []) - - if issues: - ticket_key = issues[0]["key"] - logger.info(f"Found existing Jira ticket: {ticket_key}") - return ticket_key - - summary = f"[Forge] PR #{pr_number}: {pr_title}" - logger.info(f"Creating new Jira ticket in {project_key}: {summary}") - - result = client.issue_create( - fields={ - "project": {"key": project_key}, - "summary": summary, - "issuetype": {"name": "Task"}, - } - ) - - ticket_key = result["key"] - logger.info(f"Created Jira ticket: {ticket_key}") - - return ticket_key - - -def markdown_to_adf(markdown_text): - """Convert GitHub-flavored Markdown to Atlassian Document Format. - - Args: - markdown_text: Markdown string - - Returns: - dict: ADF document ({"version": 1, "type": "doc", "content": [...]}) - """ - from marklassian import markdown_to_adf as _markdown_to_adf - - return _markdown_to_adf(markdown_text) - - -def post_comment(client, ticket_key, markdown_content, dry_run=False): - """Post a comment in ADF format to a Jira ticket. - - Args: - client: Jira client - ticket_key: Jira ticket key (e.g. PSAP-CI-42) - markdown_content: Markdown string to convert and post - dry_run: If True, log but don't post - - Returns: - bool: True if successful - """ - adf_body = markdown_to_adf(markdown_content) - - if dry_run: - logger.info(f"DRY RUN: Would post ADF comment to {ticket_key}") - return True - - logger.info(f"Posting ADF comment to {ticket_key}") - response = client.post( - f"rest/api/3/issue/{ticket_key}/comment", - data={"body": adf_body}, - ) - - if response and not isinstance(response, dict): - logger.error(f"Failed to post comment to {ticket_key}: {response}") - return False - - logger.info(f"Successfully posted comment to {ticket_key}") - return True - - -def send_jira_notification( - server, - token, - project_key, - pr_number, - pr_title, - markdown_content, - extra_tickets=None, - dry_run=False, -): - """Send a notification to Jira: find/create a ticket and post a comment. - - Args: - server: the Jira server address - token: the client token to use - project_key: Jira project key (e.g. PSAP-CI) - pr_number: GitHub PR number - pr_title: GitHub PR title - markdown_content: Markdown notification content - extra_tickets: Optional str or list of additional ticket keys to post to - dry_run: If True, log but don't post - - Returns: - bool: True if all notifications succeeded - """ - - if not server or not token: - raise ValueError("Server or token missing") - - client = get_client(server, token) - success = True - - ticket_key = find_or_create_ticket(client, project_key, pr_number, pr_title) - if not ticket_key: - logger.error("Failed to find or create Jira ticket") - return False - - if not post_comment(client, ticket_key, markdown_content, dry_run=dry_run): - success = False - - if extra_tickets: - if isinstance(extra_tickets, str): - extra_tickets = [extra_tickets] - - for extra_key in extra_tickets: - logger.info(f"Posting to extra ticket: {extra_key}") - if not post_comment(client, extra_key, markdown_content, dry_run=dry_run): - logger.warning(f"Failed to post comment to extra ticket {extra_key}") - success = False - - return success diff --git a/projects/core/notifications/send.py b/projects/core/notifications/send.py index c9bf2e2d8..06cddc8cf 100644 --- a/projects/core/notifications/send.py +++ b/projects/core/notifications/send.py @@ -21,7 +21,6 @@ def send_notification( message, github=True, - jira=False, dry_run=False, notification_vault=None, ): @@ -30,7 +29,6 @@ def send_notification( Args: message: The notification message content github: Whether to send to GitHub (default True) - jira: Whether to send to Jira (default False, requires notifications.jira config) dry_run: Whether to only log the message without sending (default False) notification_vault: Optional vault name to get notification secrets from @@ -52,10 +50,6 @@ def send_notification( logger.error( f"Cannot send GitHub notification: vault '{notification_vault}' not available" ) - if jira: - logger.error( - f"Cannot send Jira notification: vault '{notification_vault}' not available" - ) if not dry_run: return False @@ -68,68 +62,9 @@ def send_notification( ): failed = True - if jira and not _send_notification_to_jira( - vault_def, - message, - dry_run, - ): - failed = True - return not failed -def _send_notification_to_jira(vault_def, message, dry_run): - """Send a notification to Jira if configured in the project config. - - Reads notifications.jira config to determine project_key and extra_tickets. - Creates or finds a ticket matching the PR number, then posts the message - as an ADF comment. - """ - try: - from projects.core.notifications.jira import api as jira_api - - server_file = vault_def.content.get("jira-server") - token_file = vault_def.content.get("jira-token") - - if not ( - server_file.file_path - and server_file.file_path.exists() - and token_file.file_path - and token_file.file_path.exists() - ): - logger.error( - f"jira-server or jira-token files not available in the '{vault_def.name}' vault" - ) - return False - - server = server_file.file_path.read_text().strip() - token = token_file.file_path.read_text().strip() - - # should be retrieved from the configuration - pr_number = "1234" - pr_title = "dummy project" - project_key = "PSAPCI" - extra_tickets = [] - - return jira_api.send_jira_notification( - server=server, - token=token, - pr_number=pr_number, - pr_title=pr_title, - project_key=project_key, - markdown_content=message, - extra_tickets=extra_tickets, - dry_run=dry_run, - ) - - except Exception as e: - logger.error(f"Failed to send Jira notification: {e}") - return False - - -### - - def send_notification_to_github(vault_def, message, dry_run): """Send a generic notification message to GitHub.""" diff --git a/pyproject.toml b/pyproject.toml index 16af58628..51f7237de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,6 @@ dependencies = [ "jsonpath_ng", "pydantic>=2.0", "yq", - "marklassian", - "atlassian-python-api", ] [project.optional-dependencies] diff --git a/vaults/psap-forge-notifications.yaml b/vaults/psap-forge-notifications.yaml index 268857b06..73eb1ce5b 100644 --- a/vaults/psap-forge-notifications.yaml +++ b/vaults/psap-forge-notifications.yaml @@ -14,11 +14,3 @@ content: topsail-bot.slack-token: description: | Token to send slack notification from the TOPSAIL-bot - - jira-server: - description: | - Jira server URL (e.g. https://issues.redhat.com) - - jira-token: - description: | - Jira Personal Access Token for authentication From 2b51429d73d616ba60cbd90f661e1edf50eb4460 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Wed, 16 Sep 2026 21:33:15 +0200 Subject: [PATCH 0901/1072] [fournos_launcher] orchestration: fix the OCPCI notification --- projects/fournos_launcher/orchestration/config.yaml | 4 +++- projects/fournos_launcher/orchestration/submit.py | 12 +++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 60c977ba7..9375c095d 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -12,7 +12,9 @@ overrides: {} # don't touch this one directly, the config module will populate i extra_overrides: {} fournos_launcher: parallel_jobs: {} - +notifications: + vault: + name: psap-forge-notifications fournos: namespace: psap-automation kubeconfig: diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index b58cb83e3..774607a76 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -199,11 +199,7 @@ def get_ocpci_link(path, is_raw_file=False, base=None, is_dir=False): # Send notification through notification system try: - notification_vault = None - try: - notification_vault = config.project.get_config("caliper.export.notifications.vault") - except Exception: - pass + notification_vault = config.project.get_config("notifications.vault.name") ok = send_notification( message=notification_status, @@ -253,10 +249,12 @@ def get_ci_base_link(is_raw_file=False, is_dir=False): link_path = f"pr-logs/pull/{github_org}_{github_repo}/{pull_number}/{job}/{build_id}" + test_path = f"{test_name}/artifacts" + return ( - "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results/" + "https://gcs.ci.openshift.org/gcs/test-platform-results-public/" + link_path - + f"/artifacts/{test_name}", + + f"/artifacts/{test_name}/{test_path}", "", ) From c639882dc1f711f3d0fa3704c34e66770849fa6b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 04:59:00 +0200 Subject: [PATCH 0902/1072] [rhaiis] orchestration: config: add the missing vault configuration --- projects/rhaiis/orchestration/config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index eec40cd09..dcda8bbd8 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -78,7 +78,9 @@ caliper: tags: {} parameters: {} metrics: {} - + notifications: + enabled: true + vault: psap-forge-notifications agentic: enabled: true model_key: qwen-3-6-35b From c9580818e8382bcc912be687dfd1ed51c9c08a60 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 05:00:09 +0200 Subject: [PATCH 0903/1072] [mcp_gateway] orchestration: config: add the missing vault configuration --- projects/mcp_gateway/orchestration/config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 16b161358..629908e91 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -134,3 +134,6 @@ caliper: single_run: "{prefix}-s{num_servers}-u{users}-v{mcp_gateway_version}-{timestamp}" parent_run: "{prefix}-{preset}-{mcp_gateway_version}-{timestamp}" child_run: "{prefix}-s{num_servers}-u{users}-v{mcp_gateway_version}-{timestamp}" + notifications: + enabled: true + vault: psap-forge-notifications From c5a8e47a29b8d2fee9187bbba83df1075100d1e8 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 11:37:18 +0200 Subject: [PATCH 0904/1072] [core] ci_entrypoint: prepare_ci: remove warnings, make more accurate --- projects/core/ci_entrypoint/prepare_ci.py | 31 ++++++++++------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 1b775ef9e..f7243d189 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -327,31 +327,26 @@ def ci_banner(project: str, operation: str, args: list[str]): base_sha = os.environ.get("PULL_BASE_SHA", "main") if base_sha == "main": logger.info("PULL_BASE_SHA not set. Showing the last commits from main.") + pull_sha = os.environ.get("PULL_PULL_SHA", "") if not pull_sha: - logger.info("PULL_PULL_SHA not set. Showing the last commits from main.") - - logger.info(f"Git command will be: git show --quiet --oneline {base_sha}..{pull_sha}") + logger.info("PULL_PULL_SHA not set. Showing the last commits up to HEAD.") + pull_sha = "HEAD" try: - result = subprocess.run( - ["git", "show", "--quiet", "--oneline", f"{base_sha}"], - capture_output=True, - text=True, - timeout=10, - ) - logger.info(f"Git command returncode: {result.returncode}") - logger.info(f"Git stdout: {result.stdout}") - logger.info(f"Git stderr: {result.stderr}") + run.run(f"git fetch --quiet origin {base_sha}", check=False, timeout=30) - if result.returncode == 0: - lines = result.stdout.split("\n")[:10] # head 10 - for line in lines: - logger.info(line) - else: + result = run.run( + f"git show --quiet --oneline {base_sha}..{pull_sha}", + check=False, + timeout=30, + ) + if result.returncode != 0: logger.warning("Could not access git history (main..) ...") + except subprocess.TimeoutExpired: + logger.warning("Git banner commands timed out after 30s, continuing CI setup") except Exception as e: - logger.warning(f"Could not access git history: {e}") + logger.warning(f"Git banner commands failed: {e}, continuing CI setup") def system_prechecks() -> bool: From 56353f59f2e807e82f37a9365bc08d095ffb0063 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 11:42:09 +0200 Subject: [PATCH 0905/1072] [rhaiis] orchestration: test_phase: mute the 'uuid' warning --- projects/rhaiis/orchestration/test_phase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index b1d50ee89..13f29ba33 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -127,7 +127,9 @@ def _run_test( from projects.core.library import config as _cfg version = _cfg.project.get_config("tests.rhaiis.version", "") - run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "", warn=False) or str( + _uuid_mod.uuid4() + ) logger.info("Run UUID for this job: %s", run_uuid) import subprocess From ead590791c54b1c2576bef9d35794ab84003b63e Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 13:09:52 +0200 Subject: [PATCH 0906/1072] [llm_d] orchestration: ci: use vault.phase_vault_init instead of project equivalent --- projects/llm_d/orchestration/ci.py | 31 +++++++++--------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index f96e20f72..436e0783a 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -32,12 +32,9 @@ ) from projects.llm_d.orchestration.preflight_phase import run as preflight_toolbox_run from projects.llm_d.orchestration.prepare_sequence import run_prepare_sequence +from projects.rhoai.library.deploy import list_mandatory_vaults as rhoai_list_mandatory_vaults logger = logging.getLogger(__name__) -RHOAI_CUSTOM_CATALOG_VAULTS = [ - "psap-rhoai-rc", - "psap-forge-staging-image-pull", -] def init(presets=None): @@ -54,29 +51,19 @@ def init(presets=None): def list_vaults() -> list[str]: """List all vaults (includes both mandatory and optional).""" - all_vaults = vault.phase_vault_list_all() - if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): - return [*all_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] - return all_vaults + all_vaults = vault.phase_vault_list_all() + return [*all_vaults, *rhoai_list_mandatory_vaults()] -def init_vaults_for_phase(phase: str) -> None: - mandatory_vaults = [ - *config.project.get_config("vaults.all", []), - *config.project.get_config(f"vaults.{phase}", []), - ] - optional_vaults = [ - *config.project.get_config("vaults.all-optional", []), - *config.project.get_config(f"vaults.{phase}-optional", []), - ] +def init_vaults_for_phase(phase: str): + """Initialize vaults for a specific CI phase, including conditional vaults.""" - if phase == "prepare" and config.project.get_config( - "platform.rhoai.custom_catalog.enabled", False - ): - mandatory_vaults = [*mandatory_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] + extra_mandatory = [] + if phase == "prepare": + extra_mandatory.extend(rhoai_list_mandatory_vaults()) - vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) + vault.phase_vault_init(phase, extra_mandatory=(extra_mandatory or None)) @click.group(cls=ci_lib.HelpfulGroup) From a53e4a71455cd1ae4b74840e749511c09d333d66 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 13:12:00 +0200 Subject: [PATCH 0907/1072] [core] agentic: on_failure: turn warning logs into info --- projects/core/agentic/on_failure/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/core/agentic/on_failure/__init__.py b/projects/core/agentic/on_failure/__init__.py index a4d577efe..59523a2d5 100755 --- a/projects/core/agentic/on_failure/__init__.py +++ b/projects/core/agentic/on_failure/__init__.py @@ -159,7 +159,9 @@ def wrapper(*args, **kwargs): try: exit_code = func(*args, **kwargs) except Exception as e: - logger.error(f"🤖 Exception in CI command '{function_name}': {e}") + logger.info( + f"🤖 Exception in CI command '{function_name}': {e} ==> triggering failure agent" + ) _try_run_agent_on_exception() raise e @@ -169,8 +171,8 @@ def wrapper(*args, **kwargs): return exit_code # Failure - run agent - logger.warning( - f"❌ CI command '{function_name}' failed with exit code {exit_code} - triggering failure agent" + logger.info( + f"❌ CI command '{function_name}' failed with exit code {exit_code} ==> triggering failure agent" ) _run_agent_and_log_results() return exit_code From 80c263bc41cc5741b80ee124ad79b07721a6448f Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 13:17:31 +0200 Subject: [PATCH 0908/1072] [core] library: export: do not try to re-init the vault mgmt --- projects/core/library/export.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 8ca3df9ff..009ff1b76 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -210,23 +210,6 @@ def run_caliper_orchestration_export( "caliper.export.backend.mlflow.config.run_name", os.environ["FJOB_NAME"], print=False ) - # Initialize vaults needed for export operations - try: - from projects.core.library import vault - - # Get export-specific vaults (MLflow, S3, notifications) - export_vaults = caliper_export_list_vaults() - - if export_vaults: - vault.init(vaults=export_vaults) - logger.info(f"Initialized vault manager with {len(export_vaults)} vaults for export") - else: - logger.info("No vaults needed for export operation") - - except Exception as e: - logger.warning(f"Failed to initialize vaults for export: {e}") - logger.warning("Continuing with export operation - some features may not work") - caliper_cfg = config.project.get_config("caliper", print=False) return run_from_orchestration_config( From 332e48ce414cf8d87fc7e1520f1e87daf9beb625 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 13:58:32 +0200 Subject: [PATCH 0909/1072] [caliper] orchestration: censoring: rename s/discover_vault_secrets/retrieve_vault_secrets --- projects/caliper/orchestration/censoring.py | 26 +++++++++---------- projects/core/library/export_notifications.py | 2 +- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index 7009940ac..b5b6d4511 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -21,11 +21,11 @@ logger = logging.getLogger(__name__) -def discover_vault_secrets( +def retrieve_vault_secrets( verbose: bool = False, ) -> tuple[set[str], dict[str, str], dict[str, str]]: """ - Discover all vault secrets for censoring (all vault content is treated as sensitive). + Retrieve vault secrets for censoring from the initialized vaults. Args: verbose: Enable verbose logging @@ -42,13 +42,13 @@ def discover_vault_secrets( try: vault_manager = vault_lib.get_vault_manager() - available_vaults = vault_manager.list_vaults() + vault_names = vault_manager.list_initialized_vaults() if verbose: - logger.info(f"Discovering vault secrets from {len(available_vaults)} vaults") + logger.info(f"Retrieving vault secrets from {len(vault_names)} vaults") secrets_discovered = 0 - for vault_name in available_vaults: + for vault_name in vault_names: vault = vault_manager.get_vault(vault_name) if not vault: continue @@ -87,10 +87,10 @@ def discover_vault_secrets( logger.warning(f"Failed to read vault content {vault_name}/{content_name}: {e}") if verbose: - logger.info(f"Discovered {secrets_discovered} vault secrets for censoring") + logger.info(f"Retrieved {secrets_discovered} vault secrets for censoring") except Exception as e: - logger.exception(f"Failed to discover vault secrets: {e}") + logger.exception(f"Failed to retrieve vault secrets: {e}") raise return vault_secrets, secret_mapping, censor_text_mapping @@ -136,10 +136,9 @@ def censor_text(text: str, verbose: bool = False) -> str: f"Censored {len(matches)} instances of pattern '{KEYWORD_PATTERNS[i]}' in text" ) - # Now try vault secrets discovery and censoring + # Now try vault secrets retrieval and censoring try: - # Discover vault secrets for censoring - vault_secrets, secret_mapping, censor_text_mapping = discover_vault_secrets(verbose=verbose) + vault_secrets, secret_mapping, censor_text_mapping = retrieve_vault_secrets(verbose=verbose) # /!\ secret_mapping contains the secret values. Process with extra care. # Replace vault secrets (more specific) @@ -165,8 +164,8 @@ def censor_text(text: str, verbose: bool = False) -> str: logger.info("No sensitive content found in text") except Exception as e: - logger.error(f"Failed to discover vault secrets during text censoring: {e}") - # Propagate the vault discovery failure after keyword censoring is complete + logger.error(f"Failed to retrieve vault secrets during text censoring: {e}") + # Propagate the vault retrieval failure after keyword censoring is complete # This allows _censor_notification_text to handle the failure appropriately raise @@ -194,8 +193,7 @@ def orchestration_apply_censoring( logger.info("Censoring disabled via --disable-censoring flag") return False - # Discover vault secrets - vault_secrets, secret_mapping, censor_text_mapping = discover_vault_secrets( + vault_secrets, secret_mapping, censor_text_mapping = retrieve_vault_secrets( verbose=export_cfg.verbose ) # /!\ secret_mapping contains the secret values. Process with extra care. diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index ed2af7850..19d29fa7a 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -293,7 +293,7 @@ def _censor_notification_text(text: str, verbose: bool = False) -> str: Censored notification text with sensitive content replaced Raises: - Exception: If vault discovery fails, preventing notification delivery + Exception: If vault secret retrieval fails, preventing notification delivery """ try: return censor_text(text, verbose=verbose) From 2bfb370276da147edb67997b8b6ead6d7b962412 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 14:03:31 +0200 Subject: [PATCH 0910/1072] [core] library: vault: update the VaultManger to focus on initialized/requested vaults --- projects/caliper/orchestration/censoring.py | 2 +- projects/core/library/vault.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/projects/caliper/orchestration/censoring.py b/projects/caliper/orchestration/censoring.py index b5b6d4511..e4aea47e3 100644 --- a/projects/caliper/orchestration/censoring.py +++ b/projects/caliper/orchestration/censoring.py @@ -42,7 +42,7 @@ def retrieve_vault_secrets( try: vault_manager = vault_lib.get_vault_manager() - vault_names = vault_manager.list_initialized_vaults() + vault_names = vault_manager.list_vaults() if verbose: logger.info(f"Retrieving vault secrets from {len(vault_names)} vaults") diff --git a/projects/core/library/vault.py b/projects/core/library/vault.py index c4d878ef2..07f071cc5 100644 --- a/projects/core/library/vault.py +++ b/projects/core/library/vault.py @@ -81,6 +81,7 @@ class VaultManager: def __init__(self, vault_definitions_dir: Path = None): self.vault_definitions_dir = vault_definitions_dir or env.FORGE_HOME / "vaults" self._vault_cache: dict[str, VaultDefinition] = {} + self._initialized_vaults: set[str] = set() self._load_vault_definitions() def _load_vault_definitions(self): @@ -140,12 +141,14 @@ def _load_vault_definition(self, vault_file: Path) -> VaultDefinition: return vault_def def get_vault(self, vault_name: str) -> VaultDefinition | None: - """Get a vault definition by name""" + """Get a vault definition by name, only if it has been initialized""" + if vault_name not in self._initialized_vaults: + return None return self._vault_cache.get(vault_name) def list_vaults(self) -> list[str]: - """List all available vault names""" - return list(self._vault_cache.keys()) + """List initialized vault names""" + return list(self._initialized_vaults) def validate_vault(self, vault_name: str, strict: bool = True) -> bool: """ @@ -161,7 +164,7 @@ def validate_vault(self, vault_name: str, strict: bool = True) -> bool: # Override strict parameter if globally disabled global _strict_validation_enabled effective_strict = strict and _strict_validation_enabled - vault = self.get_vault(vault_name) + vault = self._vault_cache.get(vault_name) if vault is None: logger.error(f"Vault '{vault_name}' is not defined") return False @@ -331,7 +334,7 @@ def validate_all_vaults(self, strict: bool = True) -> bool: """Validate all defined vaults""" all_valid = True - for vault_name in self.list_vaults(): + for vault_name in self._vault_cache: if not self.validate_vault(vault_name, strict=strict): all_valid = False @@ -355,8 +358,7 @@ def _filter_and_validate_vaults( if strict is None: strict = _strict_validation_enabled - # Filter to only keep specified vaults, but don't remove others if this is not the first call - available_vaults = set(vault_manager.list_vaults()) + available_vaults = set(vault_manager._vault_cache) requested_vaults = set(vaults) # Check for requested vaults that don't exist @@ -370,6 +372,8 @@ def _filter_and_validate_vaults( # Remove missing vaults from requested list requested_vaults = requested_vaults - missing_vaults + vault_manager._initialized_vaults.update(requested_vaults) + logger.info( f"Processing {len(requested_vaults)} vaults with strict={strict}: {sorted(requested_vaults)}" ) From 315e69c6bdb051869610c46f83b2f8b6fdbe2ef6 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 14:06:32 +0200 Subject: [PATCH 0911/1072] [core] library: run: allow passing a timeout --- projects/core/library/run.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/core/library/run.py b/projects/core/library/run.py index b420c6af1..1986dac8e 100644 --- a/projects/core/library/run.py +++ b/projects/core/library/run.py @@ -44,6 +44,7 @@ def run( log_command=True, decode_stdout=True, decode_stderr=True, + timeout=None, ): if log_command: logger.info(f"run: {command}") @@ -64,6 +65,9 @@ def run( raise ValueError("Argument 'stdin_file' must be an open file (with a file descriptor)") args["stdin"] = stdin_file + if timeout is not None: + args["timeout"] = timeout + if protect_shell: command = f"set -o errexit;set -o pipefail;set -o nounset;set -o errtrace;{command}" From 8ccd54ea334064142e55e8190600ee01a8b9cc80 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 14:26:27 +0200 Subject: [PATCH 0912/1072] [rhoai] library: deploy: expose the required vaults --- projects/rhoai/library/deploy.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py index cefc7b28f..aa2038649 100644 --- a/projects/rhoai/library/deploy.py +++ b/projects/rhoai/library/deploy.py @@ -16,9 +16,26 @@ from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command from projects.core.dsl.utils.k8s import oc, oc_get_json from projects.core.library import vault +from projects.core.library.config import requires logger = logging.getLogger(__name__) +RHOAI_CUSTOM_CATALOG_VAULTS = [ + "psap-rhoai-rc", + "psap-forge-staging-image-pull", +] + + +@requires(custom_catalog_enabled="platform.rhoai.custom_catalog.enabled") +def list_mandatory_vaults(_cfg) -> list[str]: + """Return extra mandatory vaults needed when custom catalog is enabled.""" + + if _cfg.custom_catalog_enabled: + return list(RHOAI_CUSTOM_CATALOG_VAULTS) + + return [] + + RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" RHOAI_PULL_SECRET_NAME = "pull-secret" RHOAI_CATALOG_REGISTRIES = ("quay.io/rhoai",) From e801554c67381ed9840d4699040b858e4679e460 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 14:46:09 +0200 Subject: [PATCH 0913/1072] [llm_d] tests: test_deployment_profiles: disable the export and notification (don't initialize their vaults) --- projects/llm_d/tests/test_deployment_profiles.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py index 17197f25f..b469786ea 100644 --- a/projects/llm_d/tests/test_deployment_profiles.py +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -23,6 +23,8 @@ "caliper.postprocess.enabled": False, "cpt.kpi.labels.product_version": "RHOAI-XXX", "agentic.enabled": False, + "caliper.export.backend.mlflow.enabled": False, + "caliper.export.notifications.enabled": False, } # Check for save deployments mode via environment variable From 6bf82f26284e6b580b1d9c4d1688092b883adfbe Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 14:52:37 +0200 Subject: [PATCH 0914/1072] [core] library: export: don't request the post-processing vaults if disabled --- projects/core/library/export.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/core/library/export.py b/projects/core/library/export.py index 009ff1b76..ee21486f5 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -551,6 +551,9 @@ def caliper_export_list_vaults() -> list[str]: try: from projects.core.library import config + # S3 vaults are only needed when postprocessing is enabled + postprocess_enabled = config.project.get_config("caliper.postprocess.enabled", False) + # Check if S3 export or import is enabled in the project configuration s3_parent_config = config.project.get_config("caliper.postprocess.s3", {}) s3_export_config = config.project.get_config("caliper.postprocess.s3.export", {}) @@ -559,7 +562,7 @@ def caliper_export_list_vaults() -> list[str]: s3_export_enabled = s3_export_config.get("enabled", False) s3_import_enabled = s3_import_config.get("enabled", False) - if s3_export_enabled or s3_import_enabled: + if postprocess_enabled and (s3_export_enabled or s3_import_enabled): # Add the configured vault for S3 credentials (shared between import and export) vault_config = s3_parent_config.get("vault", {}) vault_name = ( From b46fb5d7dc07f83705c0f945d8f8481fb1b9f33d Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:15:42 +0200 Subject: [PATCH 0915/1072] [llm_d] orchestration: update the config/preset to run the short guidellm test by default --- projects/llm_d/orchestration/config.d/runtime.yaml | 2 +- projects/llm_d/orchestration/presets.d/presets.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index f614c95cd..524b9e5df 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -2,7 +2,7 @@ selected_preset: null model_name: Qwen/Qwen3-0.6B deployment_profile: approximate-prefix-cache smoke_request_key: default -benchmark_key: null +benchmark_key: short job_name: null namespace: forge-llm-d kserve: diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 393f55e0b..8d951b66d 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -12,14 +12,12 @@ deployment-distributed-default: smoke-base: runtime.model_name: Qwen/Qwen3-0.6B runtime.smoke_request_key: default - runtime.benchmark_key: null + runtime.benchmark_key: short smoke: extends: - smoke-base - deployment-approximate-prefix-cache - runtime.benchmark_key: short - platform.gateway.status_address_name: gateway-internal smoke-precise: extends: From 149e1b5cb144b236cb0354d091654ac66fc86486 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:19:56 +0200 Subject: [PATCH 0916/1072] [caliper] cli: commands: include error message for kpi_generate and visualize_cmd --- projects/caliper/cli/commands.py | 73 +++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index 76efded9e..207ad17cf 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -385,17 +385,29 @@ def visualize_cmd( verbose_parsing=verbose_parsing, ) - status.update( - { - "success": True, - "plugin_module": mod, - "output_files": paths, - "output_dir": str(output_dir), - "generated_files": len(paths), - } - ) - - click.echo("Wrote: " + ", ".join(paths)) + if not paths: + status.update( + { + "success": False, + "error": "No visualization files generated", + "plugin_module": mod, + "output_files": [], + "output_dir": str(output_dir), + "generated_files": 0, + } + ) + click.echo("visualize failed: no visualization files generated", err=True) + else: + status.update( + { + "success": True, + "plugin_module": mod, + "output_files": paths, + "output_dir": str(output_dir), + "generated_files": len(paths), + } + ) + click.echo("Wrote: " + ", ".join(paths)) except Exception as e: # noqa: BLE001 import traceback @@ -654,7 +666,7 @@ def kpi_generate( if test_dir_count == 0: status_data = { "success": False, - "message": "No test directories found - nothing to process for KPI generation", + "error": "No test directories found - nothing to process for KPI generation", "test_directories_count": 0, "test_directories": [], "excluded_test_directories": excluded_summary, @@ -693,7 +705,7 @@ def kpi_generate( if not status_details.get("success", True): status_data = { "success": False, - "message": status_details.get("message", "KPI generation failed"), + "error": status_details.get("message", "KPI generation failed"), "test_directories_count": test_dir_count, "test_directories": test_directories, "excluded_test_directories": excluded_summary, @@ -708,7 +720,7 @@ def kpi_generate( if not rows: status_data = { "success": False, - "message": "No KPIs generated", + "error": "No KPIs generated", "test_directories_count": test_dir_count, "test_directories": test_directories, "excluded_test_directories": excluded_summary, @@ -752,7 +764,7 @@ def kpi_generate( import traceback full_traceback = traceback.format_exc() - status_data = {"success": False, "message": str(e), "traceback": full_traceback} + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) click.echo(f"Full traceback:\n{full_traceback}", err=True) @@ -828,15 +840,28 @@ def kpi_csv_export( output_path=output, ) - status_data = { - "success": True, - "output_file": str(result_path), - "kpi_count": kpi_count, - "record_count": len(model.unified_result_records), - } - click.echo( - f"Generated dashboard CSV with {kpi_count} KPI rows from {len(model.unified_result_records)} records: {result_path}" - ) + if kpi_count == 0: + status_data = { + "success": False, + "error": "No CSV lines generated", + "output_file": str(result_path), + "kpi_count": 0, + "record_count": len(model.unified_result_records), + } + click.echo( + f"kpi csv-export failed: no CSV lines generated from {len(model.unified_result_records)} records", + err=True, + ) + else: + status_data = { + "success": True, + "output_file": str(result_path), + "kpi_count": kpi_count, + "record_count": len(model.unified_result_records), + } + click.echo( + f"Generated dashboard CSV with {kpi_count} KPI rows from {len(model.unified_result_records)} records: {result_path}" + ) except Exception as e: # noqa: BLE001 import traceback From 43da0ab17b27967c55c8e0eb5d8395407bb1b5eb Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:23:40 +0200 Subject: [PATCH 0917/1072] [caliper] orchestration: notification: always include the error message if available --- projects/caliper/orchestration/notification.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index 5caac102a..5e7c67a8a 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -71,6 +71,10 @@ def postprocess_get_file_link(file_path: Path | str, text: str = None) -> str: # Format step with message if available lines.append(f"- {step_emoji} {step_name_display}: `{step_data.get('status', 'unknown')}`") + error = step_data.get("error") + if error: + lines.append(f" * ⚠️ `{error}`") + message = step_data.get("message") if message: lines.append(f" * `{message}`") @@ -230,14 +234,8 @@ def _format_analyse_kpis_step(step_data: dict, get_file_link: callable | None) - step_status = step_data.get("status") - # Show error message if step failed - if step_status == "failed": - error_msg = step_data.get("error") - if error_msg: - lines.append(f" - ❌ `{error_msg}`") - # Show regression analysis results if the step was successful - elif step_status in ("success", "warning", "regression_detected"): + if step_status in ("success", "warning", "regression_detected"): # Show regression analysis results if step_data.get("regressions_detected"): lines.append(" - ❌ Regression detected") From 21f942f0e119f4c0dfccd36c833f3b481d724c4b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:24:52 +0200 Subject: [PATCH 0918/1072] [core] library: export_notifications: include log link to the post-process listing --- projects/core/library/export_notifications.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/projects/core/library/export_notifications.py b/projects/core/library/export_notifications.py index 19d29fa7a..72178895c 100644 --- a/projects/core/library/export_notifications.py +++ b/projects/core/library/export_notifications.py @@ -1210,7 +1210,7 @@ def _process_notification_files(step_dir: Path) -> list[str]: return notifications_from_files -def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: +def _extract_postprocess_status_info(artifact_dir: Path, get_file_link=None) -> list[str]: """Extract post-processing status information from POSTPROCESS_STATUS_FILENAME files. Returns: @@ -1235,6 +1235,8 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: relative_dir = relative_dir.parent dir_name = str(relative_dir) if relative_dir != Path(".") else "root" + base_directory = postprocess_data.get("base_directory", "") + # Extract overall status overall_success = postprocess_data.get("success", False) final_status = postprocess_data.get("final_status", "unknown") @@ -1250,7 +1252,13 @@ def _extract_postprocess_status_info(artifact_dir: Path) -> list[str]: status_emoji = ( "✔️" if status == "success" else "❌" if status == "failed" else "⚪" ) - step_statuses.append(f"{status_emoji} {step_name}") + log_file = step_data.get("log_file") + if log_file and get_file_link: + log_path = Path(base_directory) / log_file + step_label = get_file_link(log_path, text=step_name) + else: + step_label = step_name + step_statuses.append(f"{status_emoji} {step_label}") # Format overall line overall_emoji = "✅" if overall_success else "❌" @@ -1289,7 +1297,7 @@ def _process_step_details(step_dir: Path, mlflow_run_url: str | None = None) -> # Extract postprocess status for this specific step try: - postprocess_info = _extract_postprocess_status_info(step_dir) + postprocess_info = _extract_postprocess_status_info(step_dir, get_file_link) step_details.extend(postprocess_info) except Exception as e: logger.warning(f"Failed to extract postprocess status for step {step_dir.name}: {e}") From e284d8eb2723d7b9f0dcc563c38265890250439c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:26:04 +0200 Subject: [PATCH 0919/1072] [caliper] public: postprocess_models: provide mode details about PARSE_VISUALIZE_FAILED and KPI_PIPELINE_FAILED --- projects/caliper/orchestration/postprocess.py | 5 ++-- projects/caliper/public/postprocess_models.py | 28 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index cc1e83513..6de09e03d 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -955,7 +955,8 @@ def _run_visualize_step(self) -> None: VisualizeStepResult( status=StepStatus.FAILED, completed_at=time.time(), - detail=error_msg, + error=error_msg, + message=(status_data or {}).get("message"), exit_code=result.returncode, ), log_file, @@ -969,7 +970,7 @@ def _run_visualize_step(self) -> None: VisualizeStepResult( status=StepStatus.FAILED, completed_at=time.time(), - detail=f"{str(e)}\n{traceback.format_exc()}", + error=str(e), ), None, # No log file if we couldn't even start ) diff --git a/projects/caliper/public/postprocess_models.py b/projects/caliper/public/postprocess_models.py index 0dedbecf8..07b58705c 100644 --- a/projects/caliper/public/postprocess_models.py +++ b/projects/caliper/public/postprocess_models.py @@ -90,7 +90,8 @@ class VisualizeStepResult(BaseStepResult): output_files: list[str] = field(default_factory=list) output_dir: str | None = None generated_files: int = 0 - detail: str | None = None + error: str | None = None + message: str | None = None exit_code: int | None = None @@ -244,8 +245,20 @@ def get_failure_reason(self) -> str | None: case FinalPostprocessStatus.TEST_FAILED: return f"Test phase failed: {self.test_phase.message}" case FinalPostprocessStatus.PARSE_VISUALIZE_FAILED: + details = self._get_failed_step_details("parse", "visualize") + if details: + return f"Parse or visualize step failed: {details}" return "Parse or visualize step failed" case FinalPostprocessStatus.KPI_PIPELINE_FAILED: + details = self._get_failed_step_details( + "artifacts_to_kpis", + "artifacts_to_ai_data", + "s3_import", + "s3_export", + "analyse_kpis", + ) + if details: + return f"KPI pipeline failed: {details}" return "KPI pipeline failed" case FinalPostprocessStatus.PERFORMANCE_REGRESSION: return "Performance regression detected" @@ -254,6 +267,19 @@ def get_failure_reason(self) -> str | None: case _: return f"Unknown failure: {self.final_status}" + def _get_failed_step_details(self, *step_names: str) -> str | None: + """Extract error details from failed steps.""" + failed_details = [] + for name in step_names: + step = self.get_step_result(name) + if step and step.get("status") == "failed": + detail = step.get("error") or step.get("detail") or step.get("reason") + if detail: + failed_details.append(f"{name}: {detail}") + else: + failed_details.append(name) + return "; ".join(failed_details) if failed_details else None + # Helper functions for creating status objects From 65daa4b4b10c9ac05ee74fbceef4e97b3a4a2aed Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:31:11 +0200 Subject: [PATCH 0920/1072] [llm_d] tests: test_profiles: update with the smoke preset update --- projects/llm_d/tests/test_profiles.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index b35b7fbf9..2a032ed55 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -86,11 +86,8 @@ def test_smoke_presets_inherit_deployment_modes(preset: str, expected_deployment core_config.project.apply_preset(preset) assert runtime_config.get_deployment_profile_name() == expected_deployment assert runtime_config.get_model_name() == "Qwen/Qwen3-0.6B" - # Only the base "smoke" preset enables benchmarking via runtime.benchmark_key: short - if preset == "smoke": - assert runtime_config.get_benchmark_config() is not None - else: - assert runtime_config.get_benchmark_config() is None + # All smoke presets inherit smoke-base which sets runtime.benchmark_key: short + assert runtime_config.get_benchmark_config() is not None def test_benchmark_workloads_are_available() -> None: From 6c8e5bb220b4d6b88062c81d911483f3a4b92da0 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 15:47:23 +0200 Subject: [PATCH 0921/1072] [fournos_launcher] orchestration: config: initialize the psap-forge-notifications vault --- projects/fournos_launcher/orchestration/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 9375c095d..0c83bae9d 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -1,5 +1,6 @@ vaults: - psap-forge-ci +- psap-forge-notifications cluster: name: null From b5b930da337bc733641de7fedcd8dddf96563c77 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 16:23:02 +0200 Subject: [PATCH 0922/1072] [core] library: replot: remove the vault initialization ... --- projects/core/library/replot.py | 56 +-------------------------------- 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index 781ff73f6..817f4d5bd 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -13,12 +13,7 @@ from projects.caliper.orchestration.replot import run_replot_from_orchestration_config from projects.core.library import ci as ci_lib -from projects.core.library import config, env, vault -from projects.core.library.export import ( - caliper_agentic_list_vaults, - caliper_export_list_optional_vaults, - caliper_export_list_vaults, -) +from projects.core.library import config, env logger = logging.getLogger(__name__) @@ -40,55 +35,6 @@ def run_replot(*, artifact_directory: Path | None, keep_download_dir: bool | Non "caliper.replot.keep", False, print=False, warn=False ) - # Initialize vaults needed for replot operations - logger.info("Initializing vaults for replot operations") - try: - # Collect all required vaults using the same conditional logic as postprocess - all_vaults = [] - - # Add general project vaults - phase_vaults = vault.phase_vault_list_all() - all_vaults.extend(phase_vaults) - logger.info(f"Added {len(phase_vaults)} phase vaults: {phase_vaults}") - - # Add export-specific vaults (MLflow, S3, notifications) - export_vaults = caliper_export_list_vaults() - all_vaults.extend(export_vaults) - logger.info(f"Added {len(export_vaults)} export vaults: {export_vaults}") - - # Add agentic vaults (models) - agentic_vaults = caliper_agentic_list_vaults() - all_vaults.extend(agentic_vaults) - logger.info(f"Added {len(agentic_vaults)} agentic vaults: {agentic_vaults}") - - # Add optional export vaults (notifications) - optional_export_vaults = caliper_export_list_optional_vaults() - logger.info( - f"Found {len(optional_export_vaults)} optional export vaults: {optional_export_vaults}" - ) - - # Remove duplicates while preserving order - unique_vaults = list(dict.fromkeys(all_vaults)) - unique_optional_vaults = list(dict.fromkeys(optional_export_vaults)) - - logger.info(f"Total required vaults for replot: {len(unique_vaults)} - {unique_vaults}") - logger.info( - f"Total optional vaults for replot: {len(unique_optional_vaults)} - {unique_optional_vaults}" - ) - - # Initialize vaults with optional vaults handled separately - if unique_vaults or unique_optional_vaults: - vault.init(vaults=unique_vaults, optional_vaults=unique_optional_vaults) - logger.info( - f"Successfully initialized {len(unique_vaults)} required + {len(unique_optional_vaults)} optional vaults for replot" - ) - else: - logger.info("No vaults needed for replot operation") - - except Exception as e: - logger.warning(f"Failed to initialize vaults for replot: {e}") - logger.warning("Continuing with replot operation - some features may not work") - # Load MLflow configuration from export settings to get vault secrets (only if URL provided) vault_name = None vault_mlflow_secret = None From ddeeb7b54d588f9aa7274ff7dcb08ffd52d7f7e7 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Thu, 17 Sep 2026 17:35:11 +0200 Subject: [PATCH 0923/1072] [llm_d] orchestration: config: cpt: add the default test_harness For the KPI verification --- projects/llm_d/orchestration/config.d/cpt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_d/orchestration/config.d/cpt.yaml b/projects/llm_d/orchestration/config.d/cpt.yaml index b20c54f90..606cf82f8 100644 --- a/projects/llm_d/orchestration/config.d/cpt.yaml +++ b/projects/llm_d/orchestration/config.d/cpt.yaml @@ -1,6 +1,6 @@ kpi: labels: platform: OCP - test_harness: null + test_harness: default product_version: RHOAI # placeholder gpu_type: null From 7cda9d2d56ce1d5df016fa5311d8e8ebe76232f3 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 17 Sep 2026 14:18:17 -0400 Subject: [PATCH 0924/1072] rhaiis: let B200 use SCC-assigned fsGroup --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index be1635b3f..a73fd4501 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -24,4 +24,3 @@ b200: rhaiis.cluster_tag: "b200" rhaiis.gpu_types.nvidia: "b200" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] - benchmarks.guidellm.fs_group: 0 From 6c4cea0c7990211f4220fec702bd76de698e0d7a Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Thu, 17 Sep 2026 15:06:45 -0400 Subject: [PATCH 0925/1072] rhaiis: label MI355X accelerator correctly --- projects/rhaiis/orchestration/presets.d/clusters.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rhaiis/orchestration/presets.d/clusters.yaml b/projects/rhaiis/orchestration/presets.d/clusters.yaml index a73fd4501..9fb6982c1 100644 --- a/projects/rhaiis/orchestration/presets.d/clusters.yaml +++ b/projects/rhaiis/orchestration/presets.d/clusters.yaml @@ -17,6 +17,7 @@ old-zeus: mi355x: rhaiis.cluster_tag: "mi355x" + rhaiis.gpu_types.amd: "mi355x" rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] benchmarks.guidellm.fs_group: 0 From 36a755fc07a2c8ca2d5d260746036596d672db7b Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 18 Sep 2026 11:48:34 +0200 Subject: [PATCH 0926/1072] [core] image: Containerfile.lightweight: install the openssh-clients for reaching the VPN --- projects/core/image/Containerfile.lightweight | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/core/image/Containerfile.lightweight b/projects/core/image/Containerfile.lightweight index 221d6781d..d64d00d5e 100644 --- a/projects/core/image/Containerfile.lightweight +++ b/projects/core/image/Containerfile.lightweight @@ -8,7 +8,7 @@ LABEL io.k8s.display-name="OpenShift PSAP FORGE lightweight image" \ url="https://github.com/openshift-psap/forge" RUN dnf install -y --quiet \ - git jq vim wget curl rsync time file psmisc procps ca-certificates \ + git jq vim wget curl rsync time file psmisc procps ca-certificates openssh-clients \ python3 python3-pip python3-setuptools \ && dnf clean all \ && rm -rf /var/cache/yum From 054fccb66301d411b783730663b4143ee3ccc42c Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 18 Sep 2026 10:43:38 +0200 Subject: [PATCH 0927/1072] fournos/secrets_sync: new script to sync the vault between bitwarden<>yaml<>cluster --- fournos/secrets_sync/.gitignore | 1 + fournos/secrets_sync/README.md | 43 ++ .../sync_hashicorp_vault_secrets.py | 440 +++++++++++++ fournos/secrets_sync/sync_yaml_secrets.py | 586 ++++++++++++++++++ 4 files changed, 1070 insertions(+) create mode 100644 fournos/secrets_sync/.gitignore create mode 100644 fournos/secrets_sync/README.md create mode 100755 fournos/secrets_sync/sync_hashicorp_vault_secrets.py create mode 100755 fournos/secrets_sync/sync_yaml_secrets.py diff --git a/fournos/secrets_sync/.gitignore b/fournos/secrets_sync/.gitignore new file mode 100644 index 000000000..03cdeecef --- /dev/null +++ b/fournos/secrets_sync/.gitignore @@ -0,0 +1 @@ +secrets.yaml diff --git a/fournos/secrets_sync/README.md b/fournos/secrets_sync/README.md new file mode 100644 index 000000000..8b1403832 --- /dev/null +++ b/fournos/secrets_sync/README.md @@ -0,0 +1,43 @@ +# YAML Secrets Sync + +Synchronize Kubernetes secrets with a local YAML file. + +## Usage + +```bash +cd fournos/secrets_sync + +# Extract secrets from the cluster +uv run sync_yaml_secrets.py extract + +# Show differences between local file and the cluster +uv run sync_yaml_secrets.py diff + +# Push local file to the cluster +uv run sync_yaml_secrets.py sync + +# Preview changes without applying +uv run sync_yaml_secrets.py sync --dry-run +``` + +All commands default to `secrets.yaml` (git-ignored). Use `-n` to target a different namespace (default: `psap-secrets`). + +## YAML format + +```yaml +my-creds: + username: admin + password: s3cret +another-entry: + api-key: abc123 +``` + +Top-level keys are entry names (matching FournosJob `secretRefs`). Nested keys are the secret data. + +## Typical workflow + +1. `uv run sync_yaml_secrets.py extract` — pull current secrets +2. Edit `secrets.yaml` +3. `uv run sync_yaml_secrets.py diff` — review changes +4. `uv run sync_yaml_secrets.py sync` — push to cluster +5. **Delete `secrets.yaml`** — it contains plain-text secrets diff --git a/fournos/secrets_sync/sync_hashicorp_vault_secrets.py b/fournos/secrets_sync/sync_hashicorp_vault_secrets.py new file mode 100755 index 000000000..4097f19e2 --- /dev/null +++ b/fournos/secrets_sync/sync_hashicorp_vault_secrets.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Synchronize secrets from a HashiCorp Vault to Kubernetes. + +Reads entries from a Vault KV v2 engine and creates one Kubernetes +Opaque Secret per vault entry in the dedicated secrets namespace. +Existing secrets are updated in-place. + +Required environment variables +------------------------------ +* ``VAULT_TOKEN`` — Vault authentication token + +Optional environment variables +------------------------------ +* ``VAULT_ADDR`` — Vault server URL (default: ``https://vault.ci.openshift.org``) +* ``VAULT_SECRET_PATH`` — path within the KV engine (default: ``selfservice/psap``) +* ``VAULT_KV_MOUNT`` — KV v2 engine mount point (default: ``kv``) +* ``FOURNOS_SECRETS_NAMESPACE`` — target Kubernetes namespace (can also + use ``-n``). Defaults to ``psap-secrets``. + +Prerequisites +------------- +* ``kubectl`` / ``oc`` context pointing at the target cluster, or + in-cluster service-account credentials. + +Examples +-------- +Sync all vault entries under the configured path (using all defaults):: + + export VAULT_TOKEN="s.xxxxx" + python hacks/sync_vault_secrets.py + +Use custom vault, path, and namespace:: + + export VAULT_TOKEN="s.xxxxx" + export VAULT_ADDR="https://vault.example.com" + export VAULT_SECRET_PATH="path/to/secrets" + python hacks/sync_vault_secrets.py -n my-namespace + +Preview without touching the cluster:: + + python hacks/sync_vault_secrets.py --dry-run +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import re +import urllib.error +import urllib.request + +logger = logging.getLogger(__name__) + +LABEL_VAULT_ENTRY = "fournos.dev/vault-entry" +VAULT_SECRET_PATTERN = "vault-{entry}" + +KV_MOUNT_DEFAULT = "kv" +VAULT_ADDR_DEFAULT = "https://vault.ci.openshift.org" +VAULT_SECRET_PATH_DEFAULT = "selfservice/psap" +NAMESPACE_DEFAULT = "psap-secrets" + +LABEL_MANAGED_BY = "app.kubernetes.io/managed-by" +ANNOTATION_VAULT_ADDR = "fournos.dev/vault-addr" +ANNOTATION_VAULT_PATH = "fournos.dev/vault-path" +MANAGER_VALUE = "fournos-vault-sync" +SKIP_SYNC_KEY = "fournos_skip_sync" # vaults with this key won't be synchronized to the cluster + + +# --------------------------------------------------------------------------- +# Vault helpers +# --------------------------------------------------------------------------- + + +def _vault_request( + addr: str, + path: str, + token: str, + method: str = "GET", +) -> dict: + url = f"{addr}/v1/{path}" + req = urllib.request.Request(url, method=method) + req.add_header("X-Vault-Token", token) + logger.debug("%s %s", method, url) + try: + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as exc: + raise RuntimeError(f"Vault {method} {url} failed (HTTP {exc.code}): {exc.reason}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Vault {method} {url} failed (connection error): {exc.reason}") from exc + + +def vault_list( + addr: str, + mount: str, + path: str, + token: str, +) -> list[str]: + """List entries under a KV v2 path.""" + api_path = f"{mount}/metadata/{path}" + data = _vault_request(addr, api_path, token, method="LIST") + return data.get("data", {}).get("keys", []) + + +def vault_read( + addr: str, + mount: str, + path: str, + token: str, +) -> dict[str, str]: + """Read key-value pairs from a single KV v2 secret.""" + api_path = f"{mount}/data/{path}" + data = _vault_request(addr, api_path, token) + return data.get("data", {}).get("data", {}) + + +# --------------------------------------------------------------------------- +# Kubernetes helpers (lazy import so --dry-run works without kubeconfig) +# --------------------------------------------------------------------------- + + +def _k8s_core_api(): + from kubernetes import client + from kubernetes import config as k8s_config + + try: + k8s_config.load_kube_config() + except k8s_config.ConfigException: + k8s_config.load_incluster_config() + return client.CoreV1Api() + + +def _apply_secret( + v1, + name: str, + namespace: str, + data: dict[str, str], + vault_addr: str, + vault_full_path: str, +): + """Create or update an Opaque Secret.""" + from kubernetes.client import V1ObjectMeta, V1Secret + from kubernetes.client.exceptions import ApiException + + secret = V1Secret( + api_version="v1", + kind="Secret", + metadata=V1ObjectMeta( + name=name, + namespace=namespace, + labels={ + LABEL_MANAGED_BY: MANAGER_VALUE, + LABEL_VAULT_ENTRY: "true", + }, + annotations={ + ANNOTATION_VAULT_ADDR: vault_addr, + ANNOTATION_VAULT_PATH: vault_full_path, + }, + ), + type="Opaque", + string_data=data, + ) + + try: + v1.read_namespaced_secret(name, namespace) + except ApiException as exc: + if exc.status != 404: + raise + v1.create_namespaced_secret(namespace, secret) + logger.info("Created Secret %s/%s", namespace, name) + else: + v1.replace_namespaced_secret(name, namespace, secret) + logger.info("Updated Secret %s/%s", namespace, name) + + +def _list_managed_secrets(v1, namespace: str) -> list[str]: + """List all secrets managed by fournos-vault-sync.""" + from kubernetes.client.exceptions import ApiException + + try: + label_selector = f"{LABEL_MANAGED_BY}={MANAGER_VALUE},{LABEL_VAULT_ENTRY}=true" + secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) + return [secret.metadata.name for secret in secrets.items] + except ApiException as exc: + logger.error("Failed to list managed secrets: %s", exc) + raise + + +def _delete_secret(v1, name: str, namespace: str): + """Delete a Kubernetes Secret.""" + from kubernetes.client.exceptions import ApiException + + try: + v1.delete_namespaced_secret(name, namespace) + logger.info("Deleted Secret %s/%s", namespace, name) + except ApiException as exc: + if exc.status != 404: + logger.error("Failed to delete Secret %s/%s: %s", namespace, name, exc) + raise + + +# --------------------------------------------------------------------------- +# Core logic +# --------------------------------------------------------------------------- + + +def _is_vault_metadata(key: str) -> bool: + return key.startswith("secretsync/") + + +_SECRET_KEY_RE = re.compile(r"^[-._a-zA-Z0-9]+$") + + +def _is_valid_k8s_key(key: str) -> bool: + """Return True if *key* is a valid Kubernetes Secret data key.""" + return bool(key) and len(key) <= 253 and bool(_SECRET_KEY_RE.match(key)) + + +_DNS_1123_RE = re.compile(r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$") +_DNS_1123_MAX_LEN = 253 + + +def is_valid_k8s_name(name: str) -> bool: + """Return True if *name* is a valid DNS-1123 subdomain.""" + return bool(name) and len(name) <= _DNS_1123_MAX_LEN and _DNS_1123_RE.match(name) + + +def sync( + *, + vault_addr: str, + vault_token: str, + kv_mount: str, + secret_path: str, + namespace: str, + dry_run: bool, +) -> int: + logger.info("Listing vaults under %s/%s ...", kv_mount, secret_path) + try: + entries = vault_list(vault_addr, kv_mount, secret_path, vault_token) + except RuntimeError as exc: + logger.error("Cannot reach Vault, aborting: %s", exc) + return 1 + names = [e.rstrip("/") for e in entries if not e.endswith("/")] + if not names: + logger.warning("No vault entries found under %s/%s", kv_mount, secret_path) + return 0 + + logger.info("Found %d vault entries: %s", len(names), ", ".join(names)) + + # Always create v1 client for read operations (needed for destructive sync preview) + v1 = _k8s_core_api() + + # Track which secrets we process for destructive sync + processed_secrets = set() + + # Get list of existing managed secrets for destructive sync (always needed for preview) + existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) + logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) + + errors = 0 + skipped_count = 0 + for vault_name in names: + full_path = f"{secret_path}/{vault_name}" + + if not is_valid_k8s_name(vault_name): + logger.error( + "Vault entry %r is not a valid DNS-1123 name, skipping", + vault_name, + ) + errors += 1 + continue + + secret_name = VAULT_SECRET_PATTERN.format(entry=vault_name) + + try: + logger.info("Reading %s/%s ...", kv_mount, full_path) + kv_data = vault_read(vault_addr, kv_mount, full_path, vault_token) + except Exception: + logger.exception("Failed to read vault entry %s", full_path) + errors += 1 + continue + + if not kv_data: + logger.warning("Vault entry %s is empty, skipping", full_path) + continue + + # Check if vault contains fournos_skip_sync key - if so, skip this vault + if SKIP_SYNC_KEY in kv_data: + logger.info("Skipping vault %s (contains %s key)", vault_name, SKIP_SYNC_KEY) + skipped_count += 1 + processed_secrets.add(secret_name) + continue + + safe_data: dict[str, str] = {} + for k, v in kv_data.items(): + if _is_vault_metadata(k): + continue + if not _is_valid_k8s_key(k): + logger.error( + "Key %r in vault entry %s is not a valid K8s Secret key, skipping", + k, + full_path, + ) + errors += 1 + continue + safe_data[k] = v + + logger.info( + " %d key(s): %s", + len(safe_data), + ", ".join(safe_data.keys()), + ) + + if dry_run: + print(f"[dry-run] Would create/update Secret {namespace}/{secret_name}") + for key, value in safe_data.items(): + print(f" {key}: <{len(str(value))} chars>") + processed_secrets.add(secret_name) + continue + + try: + _apply_secret( + v1, + secret_name, + namespace, + safe_data, + vault_addr=vault_addr, + vault_full_path=f"{kv_mount}/{full_path}", + ) + processed_secrets.add(secret_name) + except Exception: + logger.exception( + "Failed to apply Secret %s/%s", + namespace, + secret_name, + ) + errors += 1 + + # Destructive sync: delete managed secrets that are no longer in vault + secrets_to_delete = existing_managed_secrets - processed_secrets + if secrets_to_delete: + logger.info( + "Found %d managed secrets not in vault, deleting: %s", + len(secrets_to_delete), + ", ".join(secrets_to_delete), + ) + + for secret_name in secrets_to_delete: + if dry_run: + print(f"[dry-run] Would delete Secret {namespace}/{secret_name}") + else: + try: + _delete_secret(v1, secret_name, namespace) + except Exception: + logger.exception("Failed to delete Secret %s/%s", namespace, secret_name) + errors += 1 + else: + logger.info("No managed secrets need to be deleted") + + if skipped_count > 0: + logger.info( + "Total vault entries skipped (contained %s): %d", + SKIP_SYNC_KEY, + skipped_count, + ) + + return 1 if errors else 0 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Sync secrets from a HashiCorp Vault to Kubernetes.", + epilog="Set VAULT_TOKEN in the environment. VAULT_ADDR and VAULT_SECRET_PATH have defaults.", + ) + parser.add_argument( + "-n", + "--namespace", + default=NAMESPACE_DEFAULT, + help=f"Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or {NAMESPACE_DEFAULT}).", + ) + parser.add_argument( + "--vault-addr", + default=os.environ.get("VAULT_ADDR", VAULT_ADDR_DEFAULT), + help=f"Vault server URL (default: $VAULT_ADDR or {VAULT_ADDR_DEFAULT}).", + ) + parser.add_argument( + "--kv-mount", + default=os.environ.get("VAULT_KV_MOUNT", KV_MOUNT_DEFAULT), + help=f"KV v2 engine mount point (default: {KV_MOUNT_DEFAULT}).", + ) + parser.add_argument( + "--secret-path", + default=os.environ.get("VAULT_SECRET_PATH", VAULT_SECRET_PATH_DEFAULT), + help=f"Path within the KV engine (default: $VAULT_SECRET_PATH or {VAULT_SECRET_PATH_DEFAULT}).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be created without touching the cluster.", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable debug logging.", + ) + + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)-5s %(message)s", + ) + logging.getLogger("kubernetes").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + vault_token = os.environ.get("VAULT_TOKEN", "") + if not vault_token: + logger.error("VAULT_TOKEN environment variable is not set") + return 1 + + return sync( + vault_addr=args.vault_addr, + vault_token=vault_token, + kv_mount=args.kv_mount, + secret_path=args.secret_path, + namespace=args.namespace, + dry_run=args.dry_run, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fournos/secrets_sync/sync_yaml_secrets.py b/fournos/secrets_sync/sync_yaml_secrets.py new file mode 100755 index 000000000..2b6285b6b --- /dev/null +++ b/fournos/secrets_sync/sync_yaml_secrets.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "kubernetes", +# "pyyaml", +# ] +# /// +"""Synchronize secrets between a YAML file and Kubernetes. + +Provides two subcommands: + +``extract`` + Read vault-entry-labelled Secrets from the cluster and write them + as a plain-text YAML file (suitable for importing into a vault + manager). + +``sync`` + Read a YAML file and create/update Kubernetes Opaque Secrets, + deleting any managed secrets not present in the file. + +YAML format +----------- +Top-level keys are entry names (the values used in FournosJob +``secretRefs``). Nested keys are the secret data:: + + my-creds: + username: admin + password: s3cret + another-entry: + api-key: abc123 + +Optional environment variables +------------------------------ +* ``FOURNOS_SECRETS_NAMESPACE`` — target Kubernetes namespace (can also + use ``-n``). Defaults to ``psap-secrets``. + +Examples +-------- +Extract current cluster secrets to a file:: + + python hacks/sync_yaml_secrets.py extract -o secrets.yaml + +Sync a YAML file to the cluster:: + + python hacks/sync_yaml_secrets.py sync secrets.yaml + +Preview without touching the cluster:: + + python hacks/sync_yaml_secrets.py sync secrets.yaml --dry-run +""" + +from __future__ import annotations + +import argparse +import base64 +import logging +import re +import sys + +import urllib3 +import yaml + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +logger = logging.getLogger(__name__) + +NAMESPACE_DEFAULT = "psap-secrets" + +LABEL_MANAGED_BY = "app.kubernetes.io/managed-by" +ANNOTATION_YAML_SOURCE = "fournos.dev/yaml-source" +MANAGER_VALUE = "fournos-vault-sync" + +LABEL_VAULT_ENTRY = "fournos.dev/vault-entry" +VAULT_SECRET_PATTERN = "vault-{entry}" +DEFAULT_YAML_FILE = "secrets.yaml" + +# --------------------------------------------------------------------------- +# Kubernetes helpers (lazy import so --dry-run works without kubeconfig) +# --------------------------------------------------------------------------- + + +def _k8s_core_api(): + from kubernetes import client + from kubernetes import config as k8s_config + + try: + k8s_config.load_kube_config() + except k8s_config.ConfigException: + k8s_config.load_incluster_config() + return client.CoreV1Api() + + +def _apply_secret( + v1, + name: str, + namespace: str, + data: dict[str, str], + yaml_source: str, +): + from kubernetes.client import V1ObjectMeta, V1Secret + from kubernetes.client.exceptions import ApiException + + secret = V1Secret( + api_version="v1", + kind="Secret", + metadata=V1ObjectMeta( + name=name, + namespace=namespace, + labels={ + LABEL_MANAGED_BY: MANAGER_VALUE, + LABEL_VAULT_ENTRY: "true", + }, + annotations={ + ANNOTATION_YAML_SOURCE: yaml_source, + }, + ), + type="Opaque", + string_data=data, + ) + + try: + v1.read_namespaced_secret(name, namespace) + except ApiException as exc: + if exc.status != 404: + raise + v1.create_namespaced_secret(namespace, secret) + logger.info("Created secret/%s -n %s", name, namespace) + else: + v1.replace_namespaced_secret(name, namespace, secret) + logger.info("Updated secret/%s -n %s", name, namespace) + + +def _list_managed_secrets(v1, namespace: str) -> list[str]: + from kubernetes.client.exceptions import ApiException + + try: + label_selector = f"{LABEL_VAULT_ENTRY}=true" + secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) + return [secret.metadata.name for secret in secrets.items] + except ApiException as exc: + logger.error("Failed to list managed secrets: %s", exc) + raise + + +def _delete_secret(v1, name: str, namespace: str): + from kubernetes.client.exceptions import ApiException + + try: + v1.delete_namespaced_secret(name, namespace) + logger.info("Deleted secret/%s -n %s", name, namespace) + except ApiException as exc: + if exc.status != 404: + logger.error("Failed to delete secret/%s -n %s: %s", name, namespace, exc) + raise + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + +_SECRET_KEY_RE = re.compile(r"^[-._a-zA-Z0-9]+$") +_DNS_1123_RE = re.compile(r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$") +_DNS_1123_MAX_LEN = 253 + + +def _is_valid_k8s_key(key: str) -> bool: + return bool(key) and len(key) <= 253 and bool(_SECRET_KEY_RE.match(key)) + + +def is_valid_k8s_name(name: str) -> bool: + return bool(name) and len(name) <= _DNS_1123_MAX_LEN and bool(_DNS_1123_RE.match(name)) + + +def _entry_name_from_secret(secret_name: str) -> str | None: + """Reverse ``settings.vault_secret_pattern`` to recover the entry name.""" + + regex = "^" + re.escape(VAULT_SECRET_PATTERN).replace(r"\{entry\}", r"(?P.+)") + "$" + m = re.match(regex, secret_name) + return m.group("entry") if m else None + + +# --------------------------------------------------------------------------- +# extract subcommand +# --------------------------------------------------------------------------- + + +def extract( + *, + namespace: str, + output: str | None, +) -> int: + v1 = _k8s_core_api() + + label_selector = f"{LABEL_VAULT_ENTRY}=true" + secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) + + if not secrets.items: + logger.warning("No vault-entry secrets found in namespace %s", namespace) + return 0 + + logger.info("Found %d vault-entry secrets in %s", len(secrets.items), namespace) + + result: dict[str, dict[str, str]] = {} + for secret in secrets.items: + name = secret.metadata.name + entry = _entry_name_from_secret(name) + if entry is None: + logger.warning( + "Secret %s does not match pattern %r, using raw name", + name, + VAULT_SECRET_PATTERN, + ) + entry = name + + decoded: dict[str, str] = {} + for k, v in (secret.data or {}).items(): + try: + decoded[k] = base64.b64decode(v).decode("utf-8") + except Exception: + logger.warning("Cannot decode key %s in secret %s, skipping", k, name) + continue + + if decoded: + result[entry] = decoded + logger.info(" %s: %d key(s)", entry, len(decoded)) + + def _str_representer(dumper, data): + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + dumper = yaml.Dumper + dumper.add_representer(str, _str_representer) + yaml_text = yaml.dump( + result, Dumper=dumper, default_flow_style=False, sort_keys=True, width=2**31 + ) + + if output: + with open(output, "w") as f: + f.write(yaml_text) + logger.info("Written to %s", output) + logger.warning( + "WARNING: %s contains plain-text secrets. Delete it when you are done!", + output, + ) + else: + sys.stdout.write(yaml_text) + + return 0 + + +# --------------------------------------------------------------------------- +# diff subcommand +# --------------------------------------------------------------------------- + + +def _read_cluster_secrets(v1, namespace: str) -> dict[str, dict[str, str]]: + """Read all vault-entry secrets from the cluster, returning {entry_name: {key: value}}.""" + label_selector = f"{LABEL_VAULT_ENTRY}=true" + secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) + + result: dict[str, dict[str, str]] = {} + for secret in secrets.items: + name = secret.metadata.name + entry = _entry_name_from_secret(name) + if entry is None: + entry = name + + decoded: dict[str, str] = {} + for k, v in (secret.data or {}).items(): + try: + decoded[k] = base64.b64decode(v).decode("utf-8") + except Exception: + decoded[k] = "" + result[entry] = decoded + return result + + +def diff( + *, + yaml_file: str, + namespace: str, +) -> int: + with open(yaml_file) as f: + yaml_data = yaml.safe_load(f) + + if not isinstance(yaml_data, dict): + logger.error("YAML file must be a mapping of entry names to key-value pairs") + return 1 + + v1 = _k8s_core_api() + cluster_data = _read_cluster_secrets(v1, namespace) + + yaml_entries = set(yaml_data.keys()) + cluster_entries = set(cluster_data.keys()) + + has_diff = False + + missing_in_cluster = yaml_entries - cluster_entries + if missing_in_cluster: + has_diff = True + for entry in sorted(missing_in_cluster): + keys = sorted(yaml_data[entry].keys()) if isinstance(yaml_data[entry], dict) else [] + print(f"+ secret not found in cluster: {entry}") + for k in keys: + print(f" + {k}") + print() + + extra_in_cluster = cluster_entries - yaml_entries + if extra_in_cluster: + has_diff = True + for entry in sorted(extra_in_cluster): + keys = sorted(cluster_data[entry].keys()) + print(f"- extra secret in cluster (not in YAML): {entry}") + for k in keys: + print(f" - {k}") + + common = yaml_entries & cluster_entries + for entry in sorted(common): + yaml_kv = yaml_data[entry] if isinstance(yaml_data[entry], dict) else {} + cluster_kv = cluster_data[entry] + + yaml_keys = set(yaml_kv.keys()) + cluster_keys = set(cluster_kv.keys()) + + entry_diffs: list[str] = [] + + for k in sorted(yaml_keys - cluster_keys): + entry_diffs.append(f" + field missing in cluster: {k}") + + for k in sorted(cluster_keys - yaml_keys): + entry_diffs.append(f" - extra field in cluster (not in YAML): {k}") + + for k in sorted(yaml_keys & cluster_keys): + yaml_val = str(yaml_kv[k]) + cluster_val = cluster_kv[k] + if yaml_val != cluster_val: + entry_diffs.append( + f" ~ field differs: {k} (<{len(yaml_val)} chars> yaml vs <{len(cluster_val)} chars> cluster)" + ) + + if entry_diffs: + has_diff = True + print(f"~ {entry}:") + for line in entry_diffs: + print(line) + + if not has_diff: + print("No differences found.") + + return 0 + + +# --------------------------------------------------------------------------- +# sync subcommand +# --------------------------------------------------------------------------- + + +def sync( + *, + yaml_file: str, + namespace: str, + dry_run: bool, + allow_delete: bool = False, +) -> int: + with open(yaml_file) as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict): + logger.error("YAML file must be a mapping of entry names to key-value pairs") + return 1 + + if not data: + logger.warning("YAML file is empty") + return 0 + + logger.info("Loaded %d entries from %s", len(data), yaml_file) + + v1 = _k8s_core_api() + processed_secrets: set[str] = set() + existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) + logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) + + errors = 0 + for entry_name, kv_data in data.items(): + entry_name = str(entry_name) + + if not is_valid_k8s_name(entry_name): + logger.error("Entry %r is not a valid DNS-1123 name, skipping", entry_name) + errors += 1 + continue + + if not isinstance(kv_data, dict): + logger.error("Entry %r value must be a mapping, skipping", entry_name) + errors += 1 + continue + + secret_name = VAULT_SECRET_PATTERN.format(entry=entry_name) + + safe_data: dict[str, str] = {} + for k, v in kv_data.items(): + if not _is_valid_k8s_key(str(k)): + logger.error( + "Key %r in entry %s is not a valid K8s Secret key, skipping", + k, + entry_name, + ) + errors += 1 + continue + safe_data[str(k)] = str(v) + + if not safe_data: + logger.warning("Entry %s has no valid keys, skipping", entry_name) + continue + + logger.info( + " %s -> %s: %d key(s): %s", + entry_name, + secret_name, + len(safe_data), + ", ".join(safe_data.keys()), + ) + + if dry_run: + print(f"[dry-run] Would create/update secret/{secret_name} -n {namespace}") + for key, value in safe_data.items(): + print(f" {key}: <{len(str(value))} chars>") + processed_secrets.add(secret_name) + continue + + try: + _apply_secret( + v1, + secret_name, + namespace, + safe_data, + yaml_source=yaml_file, + ) + processed_secrets.add(secret_name) + except Exception: + logger.exception( + "Failed to apply secret/%s -n %s", + secret_name, + namespace, + ) + errors += 1 + + secrets_to_delete = existing_managed_secrets - processed_secrets + if secrets_to_delete: + logger.info( + "Found %d managed secrets not in YAML: %s", + len(secrets_to_delete), + ", ".join(sorted(secrets_to_delete)), + ) + if not dry_run and not allow_delete: + try: + answer = input("Delete these secrets? [y/N] ").strip().lower() + except (EOFError, KeyboardInterrupt): + answer = "" + if answer != "y": + logger.info("Skipping deletion") + secrets_to_delete = set() + for secret_name in secrets_to_delete: + if dry_run: + print(f"[dry-run] Would delete secret/{secret_name} -n {namespace}") + else: + try: + _delete_secret(v1, secret_name, namespace) + except Exception: + logger.exception( + "Failed to delete secret/%s -n %s", + secret_name, + namespace, + ) + errors += 1 + else: + logger.info("No managed secrets need to be deleted") + + return 1 if errors else 0 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Sync secrets between a YAML file and Kubernetes.", + ) + parser.add_argument( + "-n", + "--namespace", + default=NAMESPACE_DEFAULT, + help=f"Target Kubernetes namespace (default: {NAMESPACE_DEFAULT}).", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable debug logging.", + ) + + sub = parser.add_subparsers(dest="command", required=True) + + # -- extract -- + p_extract = sub.add_parser( + "extract", + help="Extract cluster secrets to a YAML file.", + ) + p_extract.add_argument( + "-o", + "--output", + default=DEFAULT_YAML_FILE, + help=f"Output file path (default: {DEFAULT_YAML_FILE}).", + ) + + # -- diff -- + p_diff = sub.add_parser( + "diff", + help="Show differences between a YAML file and the cluster.", + ) + p_diff.add_argument( + "yaml_file", + nargs="?", + default=DEFAULT_YAML_FILE, + help=f"Path to the YAML secrets file (default: {DEFAULT_YAML_FILE}).", + ) + + # -- sync -- + p_sync = sub.add_parser( + "sync", + help="Sync a YAML file to the cluster.", + ) + p_sync.add_argument( + "yaml_file", + nargs="?", + default=DEFAULT_YAML_FILE, + help=f"Path to the YAML secrets file (default: {DEFAULT_YAML_FILE}).", + ) + p_sync.add_argument( + "--dry-run", + action="store_true", + help="Show what would be created/updated/deleted without touching the cluster.", + ) + p_sync.add_argument( + "--allow-delete", + action="store_true", + help="Delete managed secrets not present in the YAML file without prompting.", + ) + + args = parser.parse_args(argv) + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)-5s %(message)s", + ) + logging.getLogger("kubernetes").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + + if args.command == "extract": + return extract( + namespace=args.namespace, + output=args.output, + ) + + if args.command == "diff": + return diff( + yaml_file=args.yaml_file, + namespace=args.namespace, + ) + + if args.command == "sync": + return sync( + yaml_file=args.yaml_file, + namespace=args.namespace, + dry_run=args.dry_run, + allow_delete=args.allow_delete, + ) + + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4d16a02bc260c5483ef438f4f1af2ccaac7ea413 Mon Sep 17 00:00:00 2001 From: Kevin Pouget Date: Fri, 18 Sep 2026 15:58:26 +0200 Subject: [PATCH 0928/1072] [core] library: run: add env and handled_securely flags --- projects/core/library/run.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/projects/core/library/run.py b/projects/core/library/run.py index 1986dac8e..39a05067d 100644 --- a/projects/core/library/run.py +++ b/projects/core/library/run.py @@ -45,7 +45,12 @@ def run( decode_stdout=True, decode_stderr=True, timeout=None, + env=None, + handled_securely=False, ): + if handled_securely: + log_command = False + if log_command: logger.info(f"run: {command}") @@ -54,6 +59,9 @@ def run( args["cwd"] = cwd args["shell"] = True + if env is not None: + args["env"] = env + if capture_stdout: args["stdout"] = subprocess.PIPE if capture_stderr: @@ -71,7 +79,14 @@ def run( if protect_shell: command = f"set -o errexit;set -o pipefail;set -o nounset;set -o errtrace;{command}" - proc = subprocess.run(command, **args) + try: + proc = subprocess.run(command, **args) + except subprocess.CalledProcessError as e: + if handled_securely: + raise subprocess.CalledProcessError( + e.returncode, "