diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..b9eecce75 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,36 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + }, + { + "matcher": "Edit", + "hooks": [ + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Formatting Python code with ruff..." + }, + { + "type": "command", + "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"", + "statusMessage": "Fixing Python linting issues with ruff..." + } + ] + } + ] + } +} diff --git a/.github/steps/prepare_minimal_ubuntu_image/action.yml b/.github/steps/prepare_minimal_ubuntu_image/action.yml index ebbedd53c..4570b25fc 100644 --- a/.github/steps/prepare_minimal_ubuntu_image/action.yml +++ b/.github/steps/prepare_minimal_ubuntu_image/action.yml @@ -1,4 +1,5 @@ name: "Prepare FORGE minimal ubuntu image" +description: "Install minimal Python dependencies required for FORGE repository operations" runs: using: "composite" # <-- this is the important part diff --git a/.github/workflows/check_consistency.yml b/.github/workflows/check_consistency.yml index 04556928e..e447d86e9 100644 --- a/.github/workflows/check_consistency.yml +++ b/.github/workflows/check_consistency.yml @@ -18,11 +18,31 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/steps/prepare_minimal_ubuntu_image - name: Check that no symlink is broken run: bin/run_ansible_toolbox.py repo validate_no_broken_link + + - name: Check that all source files end with EOL + run: | + # Find all source files and check they end with newline + files_without_eol=() + + while IFS= read -r -d '' file; do + if [[ -s "$file" ]] && [[ -n "$(tail -c1 "$file")" ]]; then + files_without_eol+=("$file") + fi + done < <(find . -path './.git' -prune -o \( -name "*.py" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" -o -name "*.md" \) -type f -print0) + + if [[ ${#files_without_eol[@]} -gt 0 ]]; then + echo "❌ Files missing final newline:" + printf '%s\n' "${files_without_eol[@]}" + echo "Fix with: echo >> filename" + exit 1 + fi + echo "✅ All source files end with newline" diff --git a/.github/workflows/check_python_syntax.yml b/.github/workflows/check_python_syntax.yml index d13fd5ca4..41e9f3c40 100644 --- a/.github/workflows/check_python_syntax.yml +++ b/.github/workflows/check_python_syntax.yml @@ -10,9 +10,10 @@ jobs: max-parallel: 4 steps: - name: Use checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Check that all the files can be compiled run: | diff --git a/.github/workflows/image-push.yaml b/.github/workflows/image-push.yaml index 4cd5141f2..d4a880269 100644 --- a/.github/workflows/image-push.yaml +++ b/.github/workflows/image-push.yaml @@ -10,7 +10,9 @@ jobs: image-push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - name: Remove GitHub credentials from git config run: | diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index 3f4200572..6fea10352 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -16,7 +16,9 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/test_toolbox_dsl.yml b/.github/workflows/test_toolbox_dsl.yml index 669fdafc6..66ac3c418 100644 --- a/.github/workflows/test_toolbox_dsl.yml +++ b/.github/workflows/test_toolbox_dsl.yml @@ -20,7 +20,9 @@ jobs: PYTHONPATH: ${{ github.workspace }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcab8c11a..cc1dcab29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.16.0 hooks: - id: ruff args: [--fix] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..0d2f0c8c3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,89 @@ +# FORGE CI Framework — Agent Instructions + +## Project Overview + +FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on OpenShift clusters. +It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging. + +Key paths: +- `projects/` — individual project test harnesses +- `projects/NAME/orchestration` — upper layer of the test harness (config, prepare, test, cleanup) +- `projects/NAME/toolbox` — lower layer of the test harness (toolbox of scripts altering the state of the cluster, using the DSL language) +- `projects/NAME/test` — unit testing of the test harness +- `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers) +- `fournos/gitops/` — Tekton pipelines and GitOps base workflows +- `docs/` — project documentation + +## Security: Secret Handling (CRITICAL) + +**NEVER write secrets or sensitive data to artifact directories, logs, or files.** + +### Rules + +1. **Never persist secrets to `ARTIFACT_DIR`.** + The `oc_apply(path, manifest)` helper writes the manifest YAML to disk before applying it. + If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys), + it MUST NOT be written to any path under `env.ARTIFACT_DIR`. + +2. **Use `oc("apply", "-f", "-", input_text=..., handled_secretly=True)` for secrets.** + Apply secrets via stdin or a dedicated helper that does not persist the payload. + Never use the standard `oc_apply()` for Secret-kind manifests. + +3. **Never log secret values.** + Do not print, log, or include in error messages: passwords, tokens, bearer credentials, + TLS keys, pull-secret `.dockerconfigjson`, or any `data:`/`stringData:` field from a Secret. + +4. **Check the `kind:` field.** + Before calling `oc_apply(artifact_path, manifest)`, verify that `manifest.get("kind") != "Secret"`. + If it is a Secret, use a non-persisting apply path. + +5. **Artifact directories are public.** + Treat everything under `ARTIFACT_DIR` as if it will be published on the internet. + Only write non-sensitive operational data (CRs, logs, pod descriptions, events). + Pod/Deployment and other resources are safe to store, as they must not contain secrets. See point 9. + Kubernetes Secret objects should never be stored in the artifacts. Kubernetes Secret descriptions can be safely stored. + +7. **Use the vault module.** + Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret. + +8. **Never pass secrets via environment variables** + For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care. + +9. **Never pass secrets via Pod environment variables** + The Pod definition will be stored in public artifacts. Do no use plain text Pod environment variables to pass secrets to the Pod. Instead, use Secret resources and Pod environment variables that reference this secret. + +### Safe Patterns + +```python +# GOOD: Apply a Secret without persisting to disk +def apply_secret(manifest: dict) -> None: + """Apply a Secret manifest via stdin — nothing written to disk.""" + import yaml + from projects.core.dsl.utils.k8s import oc + + oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False)) + + +# GOOD: Guard before using oc_apply +def safe_oc_apply(artifact_path, manifest): + if manifest.get("kind") == "Secret": + raise ValueError( + f"Refusing to write Secret '{manifest['metadata']['name']}' to artifacts. " + "Use apply_secret() instead." + ) + oc_apply(artifact_path, manifest) +``` + +### Forbidden Patterns + +```python +# BAD: Writing a pull-secret to artifacts +src_dir = env.ARTIFACT_DIR / "src" +oc_apply(src_dir / "image-pull-secret.yaml", secret_manifest) # LEAKS CREDENTIALS + +# BAD: Logging secret content +logger.info(f"Applying secret: {manifest}") # LEAKS if manifest has sensitive data + +# BAD: Including tokens in error messages +raise RuntimeError(f"Failed to auth with token {token}") # LEAKS TOKEN +``` diff --git a/OWNERS b/OWNERS index 3fc847a58..ca8ee10bb 100644 --- a/OWNERS +++ b/OWNERS @@ -6,6 +6,7 @@ approvers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh reviewers: - ashishkamra - albertoperdomo2 @@ -13,3 +14,4 @@ reviewers: - sjmonson - ashtarkb - MML-coder + - Harshith-umesh diff --git a/docs/caliper/test-labels-format.md b/docs/caliper/test-labels-format.md new file mode 100644 index 000000000..bc2fbec3c --- /dev/null +++ b/docs/caliper/test-labels-format.md @@ -0,0 +1,130 @@ +# Test Labels Format (`__test_labels__.yaml`) + +## Purpose + +The `__test_labels__.yaml` file marks directories as Caliper test bases and provides metadata for test identification, grouping, and status tracking. + +## File Format + +```yaml +version: "1" +labels: + # Test distinguishing characteristics + key1: value1 + key2: value2 +kpi_labels: + # System/environment context for KPI labeling + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + # Test execution status (added at completion) + success: true|false + message: "Status description" +``` + +### Required Fields +- **`version`**: Schema version (currently `"1"`) +- **`labels`**: Key-value pairs describing test characteristics + +### Optional Fields +- **`kpi_labels`**: System/environment context labels for KPI records + - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`) + - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`) + - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`) +- **`completion`**: Test execution status + - **`success`**: `true` if succeeded, `false` if failed + - **`message`**: Human-readable status description + +## Labels vs KPI Labels + +### Labels (Test Configuration) +- **Purpose**: Distinguish different test configurations and variations +- **Usage**: Used for test filtering, grouping, and analysis +- **Examples**: `model_name`, `deployment_profile`, `guidellm_loadshape` +- **Scope**: Specific to the test configuration + +### KPI Labels (System Context) +- **Purpose**: Provide system/environment context for KPI records +- **Usage**: Applied to all KPI metrics from the test, enables cross-environment analysis +- **Examples**: `platform`, `gpu_type`, `test_harness` +- **Scope**: Describes the execution environment + +## Common Labels + +- **`model_name`**: AI/ML model name (e.g., `"llama-3.1-8b"`) +- **`deployment_profile`**: Deployment variant (e.g., `"simple-tp2-x4"`) +- **`guidellm_loadshape`**: Benchmark load pattern (e.g., `"heavy-heterogeneous"`) +- **`benchmark_type`**: Benchmark type (e.g., `"throughput"`, `"latency"`) + +## Common KPI Labels + +- **`platform`**: Target platform (e.g., `"CKS"`, `"RHOAI"`) +- **`gpu_type`**: GPU hardware type (e.g., `"H100"`, `"A100"`) +- **`test_harness`**: Testing framework (e.g., `"guidellm"`, `"vllm"`) + +## Examples + +### Basic Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple" + guidellm_loadshape: "default" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +``` + +### Completed Test +```yaml +version: "1" +labels: + model_name: "llama-3.1-8b" + deployment_profile: "simple-tp2-x4" + guidellm_loadshape: "heavy-heterogeneous" +kpi_labels: + platform: "CKS" + gpu_type: "H100" + test_harness: "guidellm" +completion: + success: true + message: "Test completed successfully" +``` + +### Failed Test +```yaml +version: "1" +labels: + model_name: "gpt-4o" + deployment_profile: "distributed" +kpi_labels: + platform: "RHOAI" + gpu_type: "A100" + test_harness: "vllm" +completion: + success: false + message: "Connection timeout to inference service" +``` + +## Usage + +- **Test Discovery**: Caliper scans for `__test_labels__.yaml` files to find test results +- **Test Analysis**: Labels enable filtering and comparative analysis of test configurations +- **KPI Labeling**: KPI labels are automatically applied to all generated KPI metrics +- **Cross-Environment Analysis**: KPI labels enable performance comparisons across different platforms/hardware +- **Report Generation**: Both labels and KPI labels provide context for charts and report organization +- **Status Tracking**: Completion field tracks test execution outcome + +## Best Practices + +- **Labels**: Use for test-specific configuration and distinguishing characteristics +- **KPI Labels**: Use for system/environment context that applies to all KPIs +- Generate both label sets early, add completion status at test end +- Use consistent, descriptive keys across related tests +- Use lowercase with underscores (e.g., `model_name`, `gpu_type`) +- Keep label schemas stable within projects +- Populate KPI labels from configuration to ensure consistency (e.g., from `cpt.kpi.labels.*`) +- Missing KPI labels should be handled gracefully with fallback values diff --git a/docs/orchestration/vaults.md b/docs/orchestration/vaults.md index edef7ad06..44a8928ce 100644 --- a/docs/orchestration/vaults.md +++ b/docs/orchestration/vaults.md @@ -116,7 +116,7 @@ from projects.core.library import vault vault.disable_strict_validation() # All subsequent vault operations will use warning mode -vault.init(['psap-forge-ci']) # Uses non-strict validation +vault.init(["psap-forge-ci"]) # Uses non-strict validation ``` **When to use non-strict mode:** @@ -140,10 +140,10 @@ vault.disable_strict_validation() # Optional: disable strict mode vault_manager = get_vault_manager() # Validate a specific vault -is_valid = vault_manager.validate_vault('psap-forge-ci') +is_valid = vault_manager.validate_vault("psap-forge-ci") # Validate all project vaults -is_valid = vault_manager.validate_project_vaults('llm_d') +is_valid = vault_manager.validate_project_vaults("llm_d") # Validate all defined vaults is_valid = vault_manager.validate_all_vaults() @@ -155,12 +155,12 @@ is_valid = vault_manager.validate_all_vaults() from projects.core.library.vault import get_vault_content_path # Get full path to vault content -kubeconfig_path = get_vault_content_path('psap-forge-ci', 'kubeconfig') +kubeconfig_path = get_vault_content_path("psap-forge-ci", "kubeconfig") # Returns: Path('/secrets/ci/kubeconfig') # Access via vault object -vault = get_vault_manager().get_vault('psap-forge-ci') -kubeconfig_path = vault.content['kubeconfig'].file_path +vault = get_vault_manager().get_vault("psap-forge-ci") +kubeconfig_path = vault.content["kubeconfig"].file_path # Returns: Path('/secrets/ci/kubeconfig') ``` @@ -239,7 +239,7 @@ In `projects/my-project/orchestration/vaults.yaml`: from projects.core.library.vault import validate_project_vaults # Validate the new vault (uses global strict validation setting) -validate_project_vaults('my-project') +validate_project_vaults("my-project") ``` ### 6. Expose the env var to the `forge_launcher` @@ -265,7 +265,7 @@ from projects.core.library import vault vault.init() # Initialize with specific vaults only -vault.init(['psap-forge-ci', 'forge-llm-d']) +vault.init(["psap-forge-ci", "forge-llm-d"]) ``` ### Runtime Usage @@ -277,13 +277,13 @@ from projects.core.library.vault import get_vault_manager vault_manager = get_vault_manager() # Validate project vaults before execution -if not vault_manager.validate_project_vaults('llm_d'): +if not vault_manager.validate_project_vaults("llm_d"): raise RuntimeError("Vault validation failed") # Get paths to vault content -vault = vault_manager.get_vault('forge-llm-d') -hf_token_path = vault.content['hf_token'].file_path -pull_secret_path = vault.content['rhoai_pull_secret'].file_path +vault = vault_manager.get_vault("forge-llm-d") +hf_token_path = vault.content["hf_token"].file_path +pull_secret_path = vault.content["rhoai_pull_secret"].file_path ``` ## Error Handling diff --git a/docs/toolbox/dsl.md b/docs/toolbox/dsl.md index 101326c51..54246707c 100644 --- a/docs/toolbox/dsl.md +++ b/docs/toolbox/dsl.md @@ -37,8 +37,7 @@ def ensure_project(args, ctx): ```python @when(lambda: some_other_task.status.return_value is True) @task -def follow_up(args, ctx): - ... +def follow_up(args, ctx): ... ``` Because the condition is called with **no arguments**, anything dynamic must come from a **closure**, module-level state, or another task’s `.status.return_value` (see `TaskResult` in `script_manager.py`). @@ -69,8 +68,7 @@ def wait_until_ready(args, ctx): ```python @retry(attempts=5, delay=2, backoff=1.5, retry_on_exceptions=True) @task -def call_flaky_api(args, ctx): - ... +def call_flaky_api(args, ctx): ... ``` If all attempts fail, `RetryFailure` is raised (wrapped in `TaskExecutionError` during `execute_tasks`, with the underlying `RetryFailure` available as `TaskExecutionError.__cause__` when that applies). diff --git a/docs/toolbox/toolbox-command-guide.md b/docs/toolbox/toolbox-command-guide.md index 3a6f688ee..37c15b24c 100644 --- a/docs/toolbox/toolbox-command-guide.md +++ b/docs/toolbox/toolbox-command-guide.md @@ -174,6 +174,7 @@ def capture_resources(args, ctx): shell.run("oc delete pod mypod", check=False) return "Cleanup completed" + @always @task def cleanup_resources(args, ctx): @@ -195,10 +196,13 @@ shell.run("oc get pods") result = shell.run("oc get pods -o json") if result.success: import json + pods = json.loads(result.stdout) # Save output to file -shell.run("oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt") +shell.run( + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-description.txt" +) # Don't fail on error shell.run("oc delete pod optional-pod", check=False) @@ -278,21 +282,16 @@ def capture_pod_status(args, ctx): # Human readable shell.run( - "oc describe pod mypod", - stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" + "oc describe pod mypod", stdout_dest=args.artifact_dir / "artifacts" / "pod-describe.txt" ) # Machine readable shell.run( - "oc get pod mypod -o yaml", - stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" + "oc get pod mypod -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pod-status.yaml" ) # For large scale tests, also capture JSON - shell.run( - "oc get pods -o json", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json" - ) + shell.run("oc get pods -o json", stdout_dest=args.artifact_dir / "artifacts" / "all-pods.json") ``` #### Source Directory (`src/`) @@ -356,7 +355,7 @@ def run(password: str): # DON'T DO THIS! #### ❌ BAD: Secrets in environment variables ```python -os.environ['SECRET_TOKEN'] = token # DON'T DO THIS! +os.environ["SECRET_TOKEN"] = token # DON'T DO THIS! # Environment variables appear in logs and process lists ``` @@ -490,10 +489,12 @@ Use descriptive task names that explain the **purpose**, not just the action: def wait_for_operator_to_be_ready(args, ctx): """Wait for operator deployment to reach ready state""" + @task def capture_failed_pod_logs(args, ctx): """Collect logs from pods that failed to start""" + @task def verify_service_endpoints_are_available(args, ctx): """Check that service endpoints are responding to requests""" @@ -523,7 +524,7 @@ def wait_for_resource_ready(args, ctx): shell.run( f"oc describe myresource {args.resource_name}", stdout_dest=args.artifact_dir / "artifacts" / "failed-resource-describe.txt", - check=False + check=False, ) raise RuntimeError(f"Failed to query resource {args.resource_name}: {result.stderr}") @@ -541,13 +542,17 @@ def capture_cluster_state(args, ctx): """Capture cluster state for post-mortem analysis""" # Capture even if tests failed - shell.run("oc get pods --all-namespaces", - stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", - check=False) + shell.run( + "oc get pods --all-namespaces", + stdout_dest=args.artifact_dir / "artifacts" / "all-pods.txt", + check=False, + ) - shell.run("oc get events --sort-by='.lastTimestamp'", - stdout_dest=args.artifact_dir / "artifacts" / "events.txt", - check=False) + shell.run( + "oc get events --sort-by='.lastTimestamp'", + stdout_dest=args.artifact_dir / "artifacts" / "events.txt", + check=False, + ) return "Cluster state captured" ``` @@ -560,9 +565,8 @@ def capture_cluster_state(args, ctx): ```python #!/usr/bin/env python3 -from projects.core.dsl import ( - always, entrypoint, execute_tasks, retry, shell, task, template -) +from projects.core.dsl import always, entrypoint, execute_tasks, retry, shell, task, template + @entrypoint def run( @@ -585,6 +589,7 @@ def run( """ return execute_tasks(locals()) + @task def validate_inputs(args, ctx): """Validate input parameters""" @@ -598,6 +603,7 @@ def validate_inputs(args, ctx): return "Inputs validated" + @task def setup_directories(args, ctx): """Create artifact directories""" @@ -606,6 +612,7 @@ def setup_directories(args, ctx): shell.mkdir("src") return "Directories created" + @task def verify_namespace_exists(args, ctx): """Ensure target namespace exists""" @@ -616,6 +623,7 @@ def verify_namespace_exists(args, ctx): return f"Namespace {args.namespace} verified" + @task def create_deployment_manifest(args, ctx): """Generate deployment manifest""" @@ -625,6 +633,7 @@ def create_deployment_manifest(args, ctx): return f"Deployment manifest created: {manifest_file}" + @task def apply_deployment(args, ctx): """Apply the deployment to cluster""" @@ -634,6 +643,7 @@ def apply_deployment(args, ctx): return f"Deployment {args.service_name} applied" + @retry(attempts=10, delay=5) @task def wait_for_deployment_ready(args, ctx): @@ -642,7 +652,7 @@ def wait_for_deployment_ready(args, ctx): result = shell.run( f"oc get deployment {args.service_name} -n {args.namespace} " f"-o jsonpath='{{.status.readyReplicas}}'", - check=False + check=False, ) if not result.success: @@ -654,6 +664,7 @@ def wait_for_deployment_ready(args, ctx): return False # Retry + @always @task def capture_deployment_status(args, ctx): @@ -663,18 +674,19 @@ def capture_deployment_status(args, ctx): shell.run( f"oc describe deployment {args.service_name} -n {args.namespace}", stdout_dest=args.artifact_dir / "artifacts" / "deployment-describe.txt", - check=False + check=False, ) # Capture pod status shell.run( f"oc get pods -l app={args.service_name} -n {args.namespace} -o yaml", stdout_dest=args.artifact_dir / "artifacts" / "pods-status.yaml", - check=False + check=False, ) return "Deployment status captured" + if __name__ == "__main__": run.main() ``` diff --git a/fournos/gitops/README.md b/fournos/gitops/README.md index 5f1821b3e..4eae41404 100644 --- a/fournos/gitops/README.md +++ b/fournos/gitops/README.md @@ -37,12 +37,52 @@ gitops/ - **Build**: Shipwright build configuration for building forge image from source - **BuildRun**: Triggers the build process -### Workflows -- **Task**: `forge-step` - Reusable Tekton task for executing forge commands -- **Pipelines**: Multiple pipeline variants: - - `forge-full`: Complete pipeline with pre-cleanup, prepare, test, export-artifacts, and post-cleanup - - `pipeline-test-only`: Test execution only - - `pipeline-replot`: Replotting functionality +### Workflows +- **Task**: `forge-step` — reusable Tekton task for executing forge commands +- **Pipelines**: + +| Pipeline | Steps | Finally | Use case | +|---|---|---|---| +| `forge-full` | pre-cleanup → prepare → preflight → test | post-cleanup → export-artifacts | Full lifecycle with cleanup on both ends | +| `forge-prepare-test` | prepare → preflight → test | export-artifacts | Prepare + test without cleanup | +| `forge-test-only` | preflight → test | export-artifacts | Skip preparation, just run the test | +| `forge-prepare-only` | prepare → preflight | export-artifacts | Set up the cluster without benchmarks | +| `forge-replot` | replot | export-artifacts | Re-run visualization/postprocessing | + +Select a pipeline via `spec.pipeline` in the FournosJob CR (default: `fournos-full`). + +### FournosJob `spec.env` variables + +The `forge-step` task bootstrap script reads certain well-known variables +from `spec.env` and acts on them before launching the execution engine. + +#### Git checkout + +| Variable | Description | +|---|---| +| `PULL_PULL_SHA` | Checkout a specific git commit SHA. The task runs `git fetch origin ` and `git reset --hard FETCH_HEAD` before executing. | +| `PULL_NUMBER` | Checkout the HEAD of a GitHub PR (`refs/pull//head`). Also enables GitHub PR notifications — test results are posted as a comment on the PR. | +| `PULL_BASE_SHA` | Base branch commit SHA (informational context for the execution engine). | + +If neither `PULL_PULL_SHA` nor `PULL_NUMBER` is set, the execution engine +runs with the code baked into the container image. + +`PULL_NUMBER` is the most convenient option for testing a PR: it checks +out the latest code from that PR **and** posts results back to GitHub. + +#### Example + +```yaml +spec: + env: + PULL_NUMBER: "118" # checkout PR #118 HEAD + post results to PR +``` + +```yaml +spec: + env: + PULL_PULL_SHA: "abc123" # checkout exact commit, no PR notification +``` ## Environments diff --git a/fournos/gitops/base/workflows/kustomization.yaml b/fournos/gitops/base/workflows/kustomization.yaml index d56669a87..2bdff8912 100644 --- a/fournos/gitops/base/workflows/kustomization.yaml +++ b/fournos/gitops/base/workflows/kustomization.yaml @@ -5,8 +5,12 @@ resources: # - resolve-job.yaml - task-forge-step.yaml - pipeline-full.yaml - - pipeline-test-only.yaml + - pipeline-launcher.yaml + - pipeline-nightly.yaml + - pipeline-prepare-only.yaml + - pipeline-prepare-test.yaml - pipeline-replot.yaml + - pipeline-test-only.yaml commonLabels: app.kubernetes.io/name: forge-on-fournos diff --git a/fournos/gitops/base/workflows/pipeline-launcher.yaml b/fournos/gitops/base/workflows/pipeline-launcher.yaml new file mode 100644 index 000000000..bca8f0fe8 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-launcher.yaml @@ -0,0 +1,77 @@ +# Single-step pipeline: run only (no prepare, no cleanup). +# Use on pre-configured clusters where setup/teardown is handled externally. +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: forge-launcher + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-preflight + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: preflight + - name: job-step-name + value: 00__preflight + + - name: 01-launcher + runAfter: [00-preflight] + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: launcher + - name: job-step-name + value: 01__launcher + + finally: + - name: 02-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 02__export-artifacts diff --git a/fournos/gitops/base/workflows/pipeline-nightly.yaml b/fournos/gitops/base/workflows/pipeline-nightly.yaml new file mode 100644 index 000000000..f39b7de29 --- /dev/null +++ b/fournos/gitops/base/workflows/pipeline-nightly.yaml @@ -0,0 +1,63 @@ +# Nightly pipeline: clusterless job that checks for new image versions +# and conditionally triggers a full test FournosJob. +# +# Single step that resolves the latest version, compares against the last +# tested version, and creates a FournosJob if a new version is detected. +# Error visibility is handled via notification files in the artifact directory. +# +# This pipeline runs as a clusterless FournosJob (no target cluster needed). +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: nightly + annotations: + fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:latest +spec: + workspaces: + - name: artifacts + params: + - name: fjob-name + type: string + - name: fournos-workload-namespace + type: string + - name: kubeconfig-secret + type: string + tasks: + - name: 00-nightly + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: nightly + - name: job-step-name + value: 00__nightly + + finally: + - name: 01-export-artifacts + taskRef: + name: forge-step + kind: Task + workspaces: + - name: artifacts + workspace: artifacts + params: + - name: fjob-name + value: "$(params.fjob-name)" + - name: fournos-workload-namespace + value: "$(params.fournos-workload-namespace)" + - name: kubeconfig-secret + value: "$(params.kubeconfig-secret)" + - name: job-step + value: export-artifacts + - name: job-step-name + value: 01__export-artifacts diff --git a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml index 0070f0b0b..40a5a0216 100644 --- a/fournos/gitops/base/workflows/pipeline-prepare-test.yaml +++ b/fournos/gitops/base/workflows/pipeline-prepare-test.yaml @@ -57,7 +57,7 @@ spec: value: 02__preflight - name: 03-test - runAfter: [01-preflight] + runAfter: [02-preflight] taskRef: name: forge-step kind: Task diff --git a/fournos/gitops/base/workflows/pipeline-replot.yaml b/fournos/gitops/base/workflows/pipeline-replot.yaml index 952407f02..07fdc5b92 100644 --- a/fournos/gitops/base/workflows/pipeline-replot.yaml +++ b/fournos/gitops/base/workflows/pipeline-replot.yaml @@ -17,7 +17,7 @@ spec: - name: kubeconfig-secret type: string tasks: - - name: 000-replot + - name: 00-replot taskRef: name: forge-step kind: Task @@ -34,10 +34,10 @@ spec: - name: job-step value: replot - name: job-step-name - value: 000__replot + value: 00__replot finally: - - name: 001-export-artifacts + - name: 01-export-artifacts taskRef: name: forge-step kind: Task @@ -54,4 +54,4 @@ spec: - name: job-step value: export-artifacts - name: job-step-name - value: 001__export-artifacts + value: 01__export-artifacts diff --git a/fournos/gitops/base/workflows/task-forge-step.yaml b/fournos/gitops/base/workflows/task-forge-step.yaml index dcb29b30c..d8d9e0fc9 100644 --- a/fournos/gitops/base/workflows/task-forge-step.yaml +++ b/fournos/gitops/base/workflows/task-forge-step.yaml @@ -13,6 +13,8 @@ spec: type: string - name: kubeconfig-secret type: string + default: "fournos-clusterless-placeholder" + description: "Kubeconfig secret name. Uses placeholder for clusterless jobs." - name: job-step type: string - name: job-step-name @@ -31,6 +33,8 @@ spec: env: - name: TARGET_KUBECONFIG value: /var/run/secrets/fournos-kubeconfig/kubeconfig + - name: KUBECONFIG_SECRET_PARAM + value: "$(params.kubeconfig-secret)" - name: FJOB_NAME value: "$(params.fjob-name)" - name: FOURNOS_WORKLOAD_NAMESPACE @@ -73,7 +77,21 @@ spec: # switching KUBECONFIG to the target cluster. oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - export KUBECONFIG=$TARGET_KUBECONFIG + # Check if we have a kubeconfig for target cluster + if [[ "$KUBECONFIG_SECRET_PARAM" != "fournos-clusterless-placeholder" && -f "$TARGET_KUBECONFIG" ]]; then + echo "Setting up target cluster kubeconfig..." + export KUBECONFIG=$TARGET_KUBECONFIG + export CLUSTERLESS_MODE="false" + + else + if [[ "$KUBECONFIG_SECRET_PARAM" == "fournos-clusterless-placeholder" ]]; then + echo "Running in clusterless mode - no target cluster kubeconfig" + else + echo "WARNING: Expected kubeconfig not found at: $TARGET_KUBECONFIG" + echo "Running in clusterless mode as fallback" + fi + export CLUSTERLESS_MODE="true" + fi # Function to conditionally export environment variables from FournosJob spec export_env_var_from_fjob() { @@ -88,10 +106,10 @@ spec: fi } - # Export environment variables from FournosJob spec - export_env_var_from_fjob PULL_BASE_SHA - export_env_var_from_fjob PULL_PULL_SHA - export_env_var_from_fjob PULL_NUMBER + # Export all environment variables from FournosJob spec.env + for var_name in $(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r '.spec.env // {} | keys[]'); do + export_env_var_from_fjob "$var_name" + done if [[ "${PULL_PULL_SHA:-}" ]]; then echo @@ -126,3 +144,4 @@ spec: - name: kubeconfig secret: secretName: $(params.kubeconfig-secret) + optional: true diff --git a/projects/agentic_tools/ci_base.py b/projects/agentic_tools/ci_base.py index d868f6fc6..22247beb3 100644 --- a/projects/agentic_tools/ci_base.py +++ b/projects/agentic_tools/ci_base.py @@ -133,7 +133,7 @@ def list_vaults(self) -> list[str]: return list(dict.fromkeys(all_vaults)) def should_init_vaults(self, phase: str) -> bool: - """Return False to skip vault init for certain phases.""" + """Return False to skip vault init for the resolve phase.""" return phase != "resolve-fournos-config" def register_extra_commands(self, group: click.Group) -> None: @@ -176,6 +176,13 @@ class path like ``projects.foo.notifications.MyProvider``) and # Build # ------------------------------------------------------------------ + _NIGHTLY_PHASES: dict[str, PhaseSpec] = { + "nightly": PhaseSpec( + func="projects.core.nightly.handler:run", + help="Nightly: check for new image version and trigger test if needed.", + ), + } + def build(self) -> click.Group: """Assemble and return the fully-wired click.Group.""" app = self @@ -200,6 +207,10 @@ def main(ctx): for cmd_name, spec in self.phases.items(): _register_phase_command(main, cmd_name, spec) + for cmd_name, spec in self._NIGHTLY_PHASES.items(): + if cmd_name not in self.phases: + _register_phase_command(main, cmd_name, spec) + main.add_command( create_fournos_resolve_entrypoint( vault_list_func=self.list_vaults, diff --git a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py index c70ed336b..78504f9be 100644 --- a/projects/agentic_tools/locust/toolbox/generate_prompts/main.py +++ b/projects/agentic_tools/locust/toolbox/generate_prompts/main.py @@ -237,16 +237,15 @@ def capture_logs(args, ctx): artifacts_dir = env.ARTIFACT_DIR / "artifacts" / "prompts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc( + oc( "logs", f"job/{GENERATOR_JOB_NAME}", "-n", args.namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "generator.log", ) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "generator.log").write_text(result.stdout, encoding="utf-8") return "Logs captured" diff --git a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py index 6b65ecb11..19baedb65 100644 --- a/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py +++ b/projects/agentic_tools/mcp/toolbox/deploy_mock_servers/main.py @@ -292,6 +292,4 @@ def _generate_server_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) diff --git a/projects/caliper/cli/commands.py b/projects/caliper/cli/commands.py index f06881acf..849d2f0d7 100644 --- a/projects/caliper/cli/commands.py +++ b/projects/caliper/cli/commands.py @@ -6,13 +6,13 @@ from pathlib import Path import click +import yaml -from projects.caliper.cli.s3_import import run_s3_import +from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params from projects.caliper.engine.ai_eval import run_ai_eval_export from projects.caliper.engine.file_export.artifacts_export_run import run_artifacts_export from projects.caliper.engine.file_export.artifacts_import_run import run_artifacts_import from projects.caliper.engine.file_export.mlflow_config import load_mlflow_config_yaml -from projects.caliper.engine.kpi.analyze import run_analyze from projects.caliper.engine.kpi.generate import run_kpi_generate from projects.caliper.engine.kpi.import_export import ( import_kpis_snapshot, @@ -77,10 +77,10 @@ def _plugin_tuple(ctx: click.Context): root = _root_obj(ctx) try: - mod = resolve_plugin_module_string( - root.get("plugin_cli"), - root.get("postprocess_config"), - root.get("base_dir"), + mod, _manifest_path = resolve_plugin_module_string( + base_dir=root.get("base_dir"), + postprocess_config=root.get("postprocess_config"), + cli_plugin=root.get("plugin_cli"), ) if not mod: raise RuntimeError("Plugin module not found") @@ -107,12 +107,25 @@ def _plugin_tuple(ctx: click.Context): default=True, help="Display parameter matrix summary after parsing (default: enabled).", ) +@click.option( + "--include-label", + multiple=True, + help="Include only test directories with matching labels (format: key=value).", +) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def parse_cmd( ctx: click.Context, no_cache: bool, cache_dir: Path | None, show_matrix: bool, + include_label: tuple[str, ...], + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -126,6 +139,21 @@ def parse_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + # Parse include-label options into filter dict + label_filter = None + if include_label: + label_filter = {} + for label_spec in include_label: + if "=" not in label_spec: + raise click.BadParameter( + f"Invalid label format '{label_spec}'. Use key=value format." + ) + key, value = label_spec.split("=", 1) + label_filter[key.strip()] = value.strip() + + status = {"success": False} + try: model = run_parse( base_dir=artifact_root, @@ -133,13 +161,62 @@ def parse_cmd( plugin=plugin, use_cache=not no_cache, show_parameter_matrix=show_matrix, + label_filter=label_filter, ) + + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + + status.update( + { + "success": True, + "plugin_module": mod, + "parsed_records": len(model.unified_result_records), + "test_directories": test_directories, + "test_directories_count": len(test_directories), + "cache_ref": str(model.parse_cache_ref) if model.parse_cache_ref else None, + } + ) + + click.echo( + f"Parsed {len(model.unified_result_records)} record(s) from {len(test_directories)} test directories; cache={model.parse_cache_ref}" + ) + + # Show test directories for user reference + if test_directories: + click.echo("📁 Test directories found:") + for test_dir in test_directories: + click.echo(f" • {test_dir}") + else: + click.echo("⚠️ No test directories found") + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status.update( + { + "success": False, + "error": str(e), + "traceback": full_traceback, + } + ) click.echo(f"parse failed: {e}", err=True) + click.echo("Full traceback:", err=True) + click.echo(full_traceback, err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) + + # Exit with non-zero code on failure + if not status["success"]: sys.exit(2) - click.echo( - f"Parsed {len(model.unified_result_records)} record(s); cache={model.parse_cache_ref}" - ) @click.command("visualize") @@ -154,6 +231,12 @@ def parse_cmd( type=click.Path(path_type=Path), required=True, ) +@click.option( + "--status-file", + type=click.Path(path_type=Path), + default=None, + help="Path to write status YAML for orchestration (absolute path required).", +) @click.pass_context def visualize_cmd( ctx: click.Context, @@ -163,6 +246,7 @@ def visualize_cmd( include_label: tuple[str, ...], exclude_label: tuple[str, ...], output_dir: Path, + status_file: Path | None, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, @@ -176,6 +260,9 @@ def visualize_cmd( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status = {"success": False} + try: paths = run_visualize( base_dir=artifact_root, @@ -190,11 +277,51 @@ def visualize_cmd( use_cache=True, cache_path=None, ) + + status.update( + { + "success": True, + "plugin_module": mod, + "output_files": paths, + "output_dir": str(output_dir), + "generated_files": len(paths), + } + ) + + click.echo("Wrote: " + ", ".join(paths)) + except Exception as e: # noqa: BLE001 + import traceback + + error_details = { + "error": str(e), + "traceback": traceback.format_exc(), + } + + status.update( + { + "success": False, + **error_details, + } + ) + + # Print full traceback to stderr for visibility click.echo(f"visualize failed: {e}", err=True) - sys.exit(2) + click.echo("Full traceback:", err=True) + click.echo(traceback.format_exc(), err=True) + finally: + # Write status file if requested + if status_file: + try: + status_file.parent.mkdir(parents=True, exist_ok=True) + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status, f, default_flow_style=False, sort_keys=False) + except Exception as e: + click.echo(f"Failed to write status file {status_file}: {e}", err=True) - click.echo("Wrote: " + ", ".join(paths)) + # Exit with non-zero code on failure + if not status["success"]: + sys.exit(2) @click.command("list-reports") @@ -255,6 +382,10 @@ def list_reports_cmd( @click.command("ai-eval-export", hidden=True) @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option("--no-cache", is_flag=True, help="Disable parse cache") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def ai_eval_export( ctx: click.Context, @@ -262,6 +393,8 @@ def ai_eval_export( artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + no_cache: bool, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -271,25 +404,67 @@ def ai_eval_export( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: - run_ai_eval_export(base_dir=artifact_root, plugin_module=mod, plugin=plugin, output=output) + run_ai_eval_export( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=not no_cache, + ) + status_data = {"success": True, "output_file": str(output)} + click.echo(f"Exported {output}") except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"ai-eval-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Exported {output}") # KPI commands @click.command("generate") @_workspace_cli_options @click.option("--output", type=click.Path(path_type=Path), required=True) +@click.option( + "--format", + "format_type", + type=click.Choice(["hierarchical", "jsonl"], case_sensitive=False), + default="hierarchical", + help="Output format: hierarchical JSON (default) or JSONL", +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_generate( ctx: click.Context, output: Path, + format_type: str, artifacts_dir: Path | None, postprocess_config: Path | None, plugin_module_override: str | None, + status_file: Path | None, ) -> None: _apply_workspace_cli_overrides( ctx, @@ -299,14 +474,160 @@ def kpi_generate( ) mod, plugin = _plugin_tuple(ctx) artifact_root: Path = _root_obj(ctx)["base_dir"] + + status_data = {"success": False} + try: - run_kpi_generate( - base_dir=artifact_root, plugin_module=mod, plugin=plugin, output_path=output + # First run parse to check test directory count + model = run_parse( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + use_cache=True, ) + + # Extract test directories from test nodes + test_directories = [str(node.directory) for node in model.test_nodes] + test_dir_count = len(test_directories) + + # Check if any test directories were found + if test_dir_count == 0: + status_data = { + "success": False, + "message": "No test directories found - nothing to process for KPI generation", + "test_directories_count": 0, + "test_directories": [], + } + click.echo("❌ No test directories found - KPI generation failed", err=True) + click.echo(" No __test_labels__.yaml files found in artifact directory", err=True) + + sys.exit(3) + else: + # Proceed with KPI generation + run_kpi_generate( + base_dir=artifact_root, + plugin_module=mod, + plugin=plugin, + output=output, + use_cache=True, + cache_path=None, + format_type=format_type, + ) + status_data = { + "success": True, + "output_file": str(output), + "test_directories_count": test_dir_count, + "test_directories": test_directories, + } + click.echo(f"Generated {output}") + click.echo(f"📁 Processed {test_dir_count} test directories") + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "message": str(e), "traceback": full_traceback} click.echo(f"kpi generate failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) + + +@click.command("csv-export") +@_workspace_cli_options +@click.option( + "--input", + "input_file", + type=click.Path(path_type=Path), + required=True, + help="Input KPI JSON file", +) +@click.option("--output", type=click.Path(path_type=Path), required=True, help="Output CSV file") +@click.option( + "--include-header-comments", is_flag=True, default=True, help="Include header comments in CSV" +) +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +@click.pass_context +def kpi_csv_export( + ctx: click.Context, + input_file: Path, + output: Path, + artifacts_dir: Path | None, + postprocess_config: Path | None, + plugin_module_override: str | None, + include_header_comments: bool, + status_file: Path | None, +) -> None: + _apply_workspace_cli_overrides( + ctx, + artifacts_dir=artifacts_dir, + postprocess_config=postprocess_config, + plugin_module_override=plugin_module_override, + ) + mod, plugin = _plugin_tuple(ctx) + + status_data = {"success": False} + + try: + # Read KPI file (supports both hierarchical JSON and JSONL formats) + from projects.caliper.engine.kpi.format import read_kpis_from_file + + kpi_records = read_kpis_from_file(input_file) + + # Export to CSV + from projects.caliper.engine.kpi.csv_export import export_kpis_to_csv + + result_path = export_kpis_to_csv( + plugin=plugin, + kpi_records=kpi_records, + output_path=output, + include_header_comments=include_header_comments, + ) + + status_data = { + "success": True, + "output_file": str(result_path), + "kpi_count": len(kpi_records), + } + click.echo(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") + except Exception as e: # noqa: BLE001 + import traceback + + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} + click.echo(f"kpi csv-export failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) - click.echo(f"Generated {output}") @click.command("import") @@ -321,21 +642,24 @@ def kpi_import(ctx: click.Context, snapshot: Path) -> None: click.echo(f"Imported {snapshot}") -@click.command("analyze") +@click.command("analyse-kpis") @click.option( - "--current", type=click.Path(path_type=Path), required=True, help="Current KPI file to analyze" + "--output", + type=click.Path(path_type=Path), + required=True, + help="Output file for analysis results", ) @click.option( - "--baseline-dir", + "--current-kpis-file", type=click.Path(path_type=Path), required=True, - help="Directory containing historical KPI files (will use most recent)", + help="Path to current KPIs JSON file", ) @click.option( - "--output", + "--historical-kpis-dir", type=click.Path(path_type=Path), required=True, - help="Output file for analysis results", + help="Directory containing historical KPI files", ) @click.option( "--plugin", @@ -344,73 +668,138 @@ def kpi_import(ctx: click.Context, snapshot: Path) -> None: required=True, help="Plugin module for KPI definitions and analysis rules", ) -@click.pass_context -def kpi_analyze( - ctx: click.Context, - current: Path, - baseline_dir: Path, +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) +def analyse_kpis_cmd( output: Path, + current_kpis_file: Path, + historical_kpis_dir: Path, plugin_module: str, + status_file: Path | None, ) -> None: - """Analyze KPIs for regression.""" - try: - from projects.caliper.engine.kpi.analyze import find_baseline_kpis - from projects.caliper.engine.load_plugin import load_plugin + """Analyze KPIs for orchestration (fork/exec).""" + import time + + import yaml + + # Validate paths exist before proceeding + if not current_kpis_file.exists(): + status_data = { + "success": False, + "error": f"Current KPI file not found: {current_kpis_file}", + "completed_at": time.time(), + } + click.echo(f"❌ Current KPI file not found: {current_kpis_file}", err=True) + + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + sys.exit(1) - # Load plugin for KPI definitions and analysis rules - plugin = load_plugin(plugin_module) - if not plugin: - click.echo(f"❌ Failed to load plugin: {plugin_module}", err=True) - sys.exit(1) + if not historical_kpis_dir.exists(): + status_data = { + "success": False, + "error": f"Historical KPI directory not found: {historical_kpis_dir}", + "completed_at": time.time(), + } + click.echo(f"❌ Historical KPI directory not found: {historical_kpis_dir}", err=True) - click.echo(f"🔌 Using plugin: {plugin_module}") + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + sys.exit(2) - baseline_kpis = find_baseline_kpis(baseline_dir) - if not baseline_kpis: - click.echo( - f"❌ No kpis.json files found in baseline directory: {baseline_dir}", err=True - ) - sys.exit(1) + # Delegate to engine layer + from projects.caliper.engine.kpi.analyze import analyze_kpis - click.echo(f"📊 Found {len(baseline_kpis)} baseline files to process") + result = analyze_kpis( + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + output_file=output, + plugin_module=plugin_module, + ) - # Run analysis with ALL baseline files (not just the most recent) - result = run_analyze( - current_path=current, baseline_kpis=baseline_kpis, output_path=output, plugin=plugin - ) + # Format status data for CLI + status_data = { + "success": result.get("success", False), + "completed_at": result.get("completed_at"), + } - # Check result status at CLI level - if result["status"] == "success": - click.echo(f"✅ Analysis complete. Results written to: {output}") - elif result["status"] == "skipped": - click.echo(f"⚠️ Analysis skipped: {result.get('reason', 'Unknown reason')}") - click.echo(f"📝 Output written to: {output}") - else: - click.echo(f"❌ Analysis failed: {result.get('error', 'Unknown error')}", err=True) - sys.exit(1) + if result.get("output_file"): + status_data["output_file"] = result["output_file"] - # Show the analysis results on screen - if output.exists(): - try: - import json + if result.get("error"): + status_data["error"] = result["error"] - with open(output) as f: - result_data = json.load(f) + # Extract detailed analysis results for status file + if result.get("success") and result.get("output_file"): + try: + import yaml + + with open(result["output_file"]) as f: + analysis_report = yaml.safe_load(f) + + # Extract key fields from the analysis report + if analysis_report: + processed = analysis_report.get("processed", {}) + tested = analysis_report.get("tested", {}) + overall = analysis_report.get("overall", {}) + results = analysis_report.get("results", []) + + status_data.update( + { + "baseline_source_count": processed.get("baseline_source_count", 0), + "tested": { + "total_kpis": tested.get("total_kpis", 0), + "regressions": tested.get("regressions", 0), + "passes": tested.get("passes", 0), + "skipped": tested.get("skipped", 0), + }, + "results": results, + "overall": { + "verdict": overall.get("verdict", "UNKNOWN"), + "regression_count": overall.get("regression_count", 0), + "total_tested": overall.get("total_tested", 0), + "total_skipped": overall.get("total_skipped", 0), + }, + } + ) + except Exception as e: + # If we can't read the analysis report, just include basic info + status_data["analysis_report_error"] = f"Could not read analysis report: {e}" - # Convert to YAML for better readability - import yaml + if result.get("result"): + status_data.update(result["result"]) # Include analysis result for debugging - yaml_output = yaml.dump(result_data, default_flow_style=False, indent=2) - click.echo("\n📊 Analysis Results:") - click.echo("=" * 50) - click.echo(yaml_output) - except Exception as e: - click.echo(f"⚠️ Could not display results: {e}") - else: - click.echo("⚠️ Output file not found") - except Exception as e: # noqa: BLE001 - raise e - click.echo(f"❌ kpi analyze failed: {e}", err=True) + # Display result + if result.get("success"): + click.echo(f"✅ KPI analysis completed: {result.get('output_file')}") + else: + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ KPI analysis failed: {error_msg}", err=True) + + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): sys.exit(3) @@ -431,6 +820,9 @@ def kpi_analyze( "--aws-credentials-file", default="aws.credentials", help="Credentials file name within vault" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def kpi_s3_import( ctx: click.Context, @@ -444,15 +836,13 @@ def kpi_s3_import( vault: str, aws_credentials_file: str, verbose: bool, + status_file: Path | None, ) -> None: """Download historical KPI and analysis data from S3.""" - from projects.caliper.orchestration.postprocess_config import ( - CaliperOrchestrationPostprocessConfig, - CaliperOrchestrationS3ImportSection, - CaliperOrchestrationS3Section, - ) from projects.core.library import vault as vault_lib + status_data = {"success": False} + try: # Ensure output directory exists output_dir.mkdir(parents=True, exist_ok=True) @@ -466,47 +856,85 @@ def kpi_s3_import( # Initialize vault system vault_lib.init(vaults=[vault] if vault else []) - # Parse bucket/prefix to extract instance and directory - prefix_parts = prefix.strip("/").split("/") - instance = prefix_parts[0] if len(prefix_parts) > 0 and prefix_parts[0] else None - directory = prefix_parts[1] if len(prefix_parts) > 1 else None - - s3_import_config = CaliperOrchestrationS3ImportSection( - enabled=True, - output_dir=output_dir.name, + # Run S3 import + result = run_s3_import_with_explicit_params( + bucket=bucket, + prefix=prefix, + output_dir=output_dir, + vault=vault, + aws_credentials_file=aws_credentials_file, include_kpis_json=include_kpis_json, include_kpis_csv=include_kpis_csv, include_ai_data=include_ai_data, max_downloads=max_downloads, ) - s3_config = CaliperOrchestrationS3Section( - bucket=bucket, - instance=instance, - directory=directory, - vault=vault, - aws_credentials_file=aws_credentials_file, - **{"import": s3_import_config}, - ) - - config = CaliperOrchestrationPostprocessConfig(s3=s3_config) - - # Run S3 import - result = run_s3_import(config, output_dir.parent) + # Construct S3 path for status + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" if result["status"] == "success": + downloaded_count = result.get("downloaded_files", 0) + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": downloaded_count, + "file_count": downloaded_count, + "imported_path": s3_path, + } click.echo("✅ S3 import completed successfully") - downloaded_files = result.get("files", []) - click.echo(f"📄 Downloaded {len(downloaded_files)} files to {output_dir}") - elif result["status"] == "empty": - click.echo(f"⚠️ No objects found matching filters in s3://{bucket}/{prefix}") + click.echo(f"📄 Downloaded {downloaded_count} files to {output_dir}") + elif result["status"] == "warning": + status_data = { + "success": True, + "output_dir": str(output_dir), + "downloaded_files": 0, + "file_count": 0, + "warning": result.get("message", "No objects found matching filters"), + "imported_path": s3_path, + } + click.echo(f"⚠️ {result.get('message', 'No objects found matching filters')}") else: - click.echo(f"❌ S3 import failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg, "imported_path": s3_path} + click.echo(f"❌ S3 import failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) - except Exception as e: - click.echo(f"❌ S3 import failed: {e}", err=True) - sys.exit(2) + except Exception as e: # noqa: BLE001 + import traceback + + # Construct S3 path for error cases too + s3_path = f"s3://{bucket}" + if prefix: + s3_path += f"/{prefix}" + + full_traceback = traceback.format_exc() + status_data = { + "success": False, + "error": str(e), + "traceback": full_traceback, + "imported_path": s3_path, + } + click.echo(f"S3 import failed: {e}", err=True) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + + if not status_file: + sys.exit(3) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) # Artifacts commands diff --git a/projects/caliper/cli/kpi_analysis.py b/projects/caliper/cli/kpi_analysis.py index 7acc5ca84..4eeee8d77 100644 --- a/projects/caliper/cli/kpi_analysis.py +++ b/projects/caliper/cli/kpi_analysis.py @@ -33,21 +33,34 @@ ) def analyze_cli(current: Path, historical_dir: Path, output: Path, plugin_module: str) -> None: """CLI entrypoint for KPI analysis.""" - from projects.caliper.engine.kpi.analyze import run_kpi_analysis + from projects.caliper.engine.kpi.analyze import ( + analyze_kpis_with_format_conversion, + status_dict_to_exit_code, + ) try: - # Call the core analysis function from engine - exit_code = run_kpi_analysis( - current_kpi_file=current, - historical_data_dir=historical_dir, + # Call the core analysis function from engine (returns status dict) + result = analyze_kpis_with_format_conversion( + current_kpis_file=current, + historical_kpis_dir=historical_dir, output_file=output, plugin_module=plugin_module, ) - if exit_code == 0: + # Convert status dict to exit code for CLI + exit_code = status_dict_to_exit_code(result) + + if result.get("success"): click.echo(f"✅ Analysis completed successfully. Results written to: {output}") + if result.get("regressions_detected"): + click.echo("⚠️ Regressions detected in analysis results") + elif result.get("message"): + click.echo(f"ℹ️ {result.get('message')}") else: - click.echo(f"❌ Analysis failed with exit code {exit_code}", err=True) + error_msg = result.get("error", "Unknown error") + click.echo(f"❌ Analysis failed: {error_msg}", err=True) + + if exit_code != 0: sys.exit(exit_code) except Exception as e: diff --git a/projects/caliper/cli/main.py b/projects/caliper/cli/main.py index 167b32dc3..f3e16902d 100644 --- a/projects/caliper/cli/main.py +++ b/projects/caliper/cli/main.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import sys from pathlib import Path from urllib.parse import parse_qs, urlparse @@ -10,9 +11,10 @@ from projects.caliper.cli.commands import ( ai_eval_export, + analyse_kpis_cmd, artifacts_export, artifacts_import, - kpi_analyze, + kpi_csv_export, kpi_generate, kpi_import, kpi_s3_import, @@ -148,7 +150,7 @@ def main( @main.group("kpi") @click.pass_context def kpi_group(ctx: click.Context) -> None: - """KPI generate/import/export/analyze.""" + """KPI generate/import/export/.""" @main.group("artifacts") @@ -159,6 +161,13 @@ def artifacts_group(ctx: click.Context) -> None: def run_cli() -> None: """Invoke CLI; on missing required options, print subcommand help.""" + # Configure logging to capture INFO level messages for detailed output + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + force=True, # Override any existing configuration + ) + try: # standalone_mode=False returns exit codes instead of calling sys.exit; # propagate them so failures are non-zero (e.g. ctx.exit(1) from _exit_with_help). @@ -199,8 +208,9 @@ def run_cli() -> None: # Register KPI commands kpi_group.add_command(kpi_generate) +kpi_group.add_command(kpi_csv_export) kpi_group.add_command(kpi_import) -kpi_group.add_command(kpi_analyze) +kpi_group.add_command(analyse_kpis_cmd) kpi_group.add_command(kpi_s3_import) # Register s3-export command under kpi group diff --git a/projects/caliper/cli/orchestration_entrypoints.py b/projects/caliper/cli/orchestration_entrypoints.py deleted file mode 100644 index 41203348e..000000000 --- a/projects/caliper/cli/orchestration_entrypoints.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Entrypoint functions for orchestration to call Caliper core functionality. - -This module provides well-defined interfaces between the orchestration layer -and Caliper's engine/CLI components. -""" - -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any - - -def parse_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, - use_cache: bool = True, -) -> tuple[Any, str]: - """Entrypoint for orchestration to call parse functionality.""" - from projects.caliper.engine.parse import run_parse - - mod_str, plugin = load_plugin_entrypoint(postprocess_config, tree_root, manifest_path) - - model = run_parse( - base_dir=tree_root, - plugin_module=mod_str, - plugin=plugin, - use_cache=use_cache, - ) - - return model, mod_str - - -def run_visualize_entrypoint(**kwargs) -> list[str]: - """Entrypoint for orchestration to call visualize functionality.""" - from projects.caliper.engine.visualize import run_visualize - - return run_visualize(**kwargs) - - -def load_plugin_entrypoint( - postprocess_config, - tree_root: Path, - manifest_path: Path | None, -) -> tuple[str, Any]: - """Entrypoint for orchestration to load plugins.""" - from projects.caliper.engine.load_plugin import load_plugin - from projects.caliper.engine.plugin_config import resolve_plugin_module_string - - mod_str, _manifest = resolve_plugin_module_string( - base_dir=tree_root, - postprocess_config=manifest_path, - cli_plugin=postprocess_config.plugin_module, - ) - - return mod_str, load_plugin(mod_str) - - -def s3_import_entrypoint( - postprocess_config, - output_dir: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 import functionality.""" - - if not postprocess_config.s3.import_.enabled: - return {"status": "disabled", "reason": "s3_import disabled", "completed_at": time.time()} - - try: - from projects.caliper.cli.s3_import import run_s3_import_with_explicit_params - from projects.caliper.orchestration.step_logging import log_s3_import_command - - # Log command to reproduce this step - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.import_ - - # Build import prefix using instance + s3.prefix + directory - import_prefix_parts = [] - if s3_parent_config.instance: - import_prefix_parts.append(s3_parent_config.instance) - if s3_parent_config.prefix: - import_prefix_parts.append(s3_parent_config.prefix.rstrip("/")) - if s3_parent_config.directory: - import_prefix_parts.append(s3_parent_config.directory) - - import_prefix = "/".join(import_prefix_parts) if import_prefix_parts else "" - import_dir = output_dir / s3_config.output_dir - - log_s3_import_command( - bucket=s3_parent_config.bucket, - prefix=import_prefix, - output_dir=import_dir, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - - # Use explicit parameters instead of config dict - result = run_s3_import_with_explicit_params( - bucket=s3_parent_config.bucket, - prefix=import_prefix, - output_dir=import_dir, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - include_kpis_json=s3_config.include_kpis_json, - include_kpis_csv=s3_config.include_kpis_csv, - include_ai_data=s3_config.include_ai_data, - max_downloads=s3_config.max_downloads, - ) - return result - - except Exception as e: - import logging - - logger = logging.getLogger(__name__) - logger.exception("S3 import failed") - return {"status": "failed", "error": str(e), "completed_at": time.time()} - - -def s3_export_entrypoint( - postprocess_config, - output_dir: Path, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, -) -> dict[str, Any]: - """Entrypoint for orchestration to call S3 export functionality with explicit paths.""" - from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths - - s3_parent_config = postprocess_config.s3 - s3_config = postprocess_config.s3.export - - return run_s3_export_with_explicit_paths( - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, - bucket=s3_parent_config.bucket, - prefix=s3_parent_config.prefix, - instance=s3_parent_config.instance, - directory=s3_parent_config.directory, - upload_id=s3_config.upload_id, - vault=s3_parent_config.vault.name, - aws_credentials_file=s3_parent_config.vault.aws_credentials_file, - dry_run=s3_config.dry_run, - ) - - -def get_kpi_functions_entrypoint(plugin_module) -> dict: - """Entrypoint for orchestration to get KPI function metadata.""" - from projects.caliper.engine.kpi.decorators import get_kpi_functions - - try: - plugin_module_obj = __import__(plugin_module, fromlist=[""]) - return get_kpi_functions(plugin_module_obj) - except (ImportError, AttributeError): - return {} - - -def analyze_kpis_entrypoint( - postprocess_config, - plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, -) -> dict[str, Any]: - """Entrypoint for orchestration to call KPI analysis.""" - from projects.caliper.engine.kpi.analyze import analyze_kpis - - return analyze_kpis( - postprocess_config=postprocess_config, - plugin_module=plugin_module, - base_dir=base_dir, - output_dir=output_dir, - current_kpis_file=current_kpis_file, - ) diff --git a/projects/caliper/cli/s3_export.py b/projects/caliper/cli/s3_export.py index c60bd9d36..5c500ea55 100644 --- a/projects/caliper/cli/s3_export.py +++ b/projects/caliper/cli/s3_export.py @@ -28,6 +28,38 @@ logger = logging.getLogger(__name__) +def build_s3_prefix( + instance: str | None = None, + directory: str | None = None, + upload_id: str | None = None, + trailing_slash: bool = True, +) -> str: + """Build S3 prefix from components, skipping empty parts. + + Args: + instance: Instance identifier (optional) + directory: Directory identifier (optional) + upload_id: Upload identifier (optional) + trailing_slash: Whether to add trailing slash (default: True) + + Returns: + Formatted S3 prefix path + """ + components = [] + if instance: + components.append(instance) + if directory: + components.append(directory) + if upload_id: + components.append(upload_id) + + if not components: + return "" + + prefix = "/".join(components) + return prefix + "/" if trailing_slash else prefix + + def list_ai_data_files(ai_data_dir: Path) -> list[Path]: """List all files in the AI evaluation export directory. @@ -224,7 +256,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path | None = None, analysis_file: Path | None = None, bucket: str, - prefix: str = "", instance: str | None = None, directory: str | None = None, upload_id: str | None = None, @@ -240,7 +271,6 @@ def run_s3_export_with_explicit_paths( ai_data_dir: Path to AI data directory to upload analysis_file: Path to analysis file to upload bucket: S3 bucket name - prefix: S3 object prefix/path instance: Instance identifier for S3 organization directory: Directory identifier for S3 organization upload_id: Custom upload identifier (uses timestamp if not provided) @@ -288,23 +318,14 @@ def run_s3_export_with_explicit_paths( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if prefix: - s3_path_components.append(prefix.rstrip("/")) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - logger.info( - f"Path structure: {instance}/{prefix.rstrip('/') if prefix else 'no-prefix'}/{directory}/{upload_id}/" - ) + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") @@ -601,26 +622,14 @@ def run_s3_export( upload_id = f"{timestamp}_{microseconds}" logger.info(f"Using generated collision-resistant timestamp ID: {upload_id}") - # Construct the full S3 path: {instance}/{prefix}/{directory}/{upload_id}/ - s3_path_components = [] - if instance: - s3_path_components.append(instance) - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix: - s3_path_components.append(s3_parent_config.prefix.rstrip("/")) - if directory: - s3_path_components.append(directory) - s3_path_components.append(upload_id) - - export_s3_prefix = "/".join(s3_path_components) + "/" + # Construct the full S3 path: {instance}/{directory}/{upload_id}/ + export_s3_prefix = build_s3_prefix( + instance=instance, directory=directory, upload_id=upload_id + ) logger.info(f"Starting S3 export to bucket: {bucket}") logger.info(f"Full S3 export path: s3://{bucket}/{export_s3_prefix}") - prefix_component = ( - s3_parent_config.prefix.rstrip("/") - if hasattr(s3_parent_config, "prefix") and s3_parent_config.prefix - else "no-prefix" - ) - logger.info(f"Path structure: {instance}/{prefix_component}/{directory}/{upload_id}/") + logger.info(f"Path structure: {export_s3_prefix}") if dry_run: logger.info("DRY RUN MODE: Files will not actually be uploaded") diff --git a/projects/caliper/cli/s3_export_cli.py b/projects/caliper/cli/s3_export_cli.py index 14853019f..82ca3643b 100644 --- a/projects/caliper/cli/s3_export_cli.py +++ b/projects/caliper/cli/s3_export_cli.py @@ -44,6 +44,9 @@ "--dry-run", is_flag=True, help="Show what would be uploaded without actually uploading" ) @click.option("-v", "--verbose", is_flag=True, help="Show detailed progress information") +@click.option( + "--status-file", type=click.Path(path_type=Path), help="YAML file to write operation status" +) @click.pass_context def s3_export_cmd( ctx: click.Context, @@ -60,16 +63,24 @@ def s3_export_cmd( aws_credentials_file: str, dry_run: bool, verbose: bool, + status_file: Path | None, ) -> None: """Upload postprocess artifacts to S3.""" + import yaml + + status_data = {"success": False} + try: # Validate that at least one file/directory is provided if not any([kpis_file, csv_file, ai_data_dir, analysis_file]): + error_msg = "At least one file or directory must be specified" + status_data = {"success": False, "error": error_msg} click.echo("❌ Error: At least one file or directory must be specified", err=True) click.echo( " Use --kpis-file, --csv-file, --ai-data-dir, or --analysis-file", err=True ) - sys.exit(1) + if not status_file: + sys.exit(1) # Import S3 functions from projects.caliper.cli.s3_export import run_s3_export_with_explicit_paths @@ -116,7 +127,6 @@ def s3_export_cmd( ai_data_dir=ai_data_dir, analysis_file=analysis_file, bucket=bucket, - prefix=prefix, instance=instance, directory=directory, upload_id=upload_id, @@ -126,10 +136,18 @@ def s3_export_cmd( ) if result["status"] == "success": + status_data = { + "success": True, + "exported_path": result.get("exported_path", ""), + "uploaded_files": result.get("uploaded_files", 0), + "total_files": result.get("total_files", 0), + "dry_run": dry_run, + } if dry_run: click.echo("✅ Dry run completed - see upload plan") if "dry_run_file" in result: click.echo(f"📋 Upload plan saved to: {result['dry_run_file']}") + status_data["dry_run_file"] = result["dry_run_file"] else: click.echo("✅ S3 export completed successfully") if "uploaded_files" in result: @@ -138,9 +156,31 @@ def s3_export_cmd( if verbose and "exported_path" in result: click.echo(f"🌍 S3 location: {result['exported_path']}") else: - click.echo(f"❌ S3 export failed: {result.get('error', 'unknown error')}", err=True) - sys.exit(1) + error_msg = result.get("error", "unknown error") + status_data = {"success": False, "error": error_msg} + click.echo(f"❌ S3 export failed: {error_msg}", err=True) + if not status_file: + sys.exit(1) + + except Exception as e: # noqa: BLE001 + import traceback - except Exception as e: + full_traceback = traceback.format_exc() + status_data = {"success": False, "error": str(e), "traceback": full_traceback} click.echo(f"❌ S3 export failed: {e}", err=True) - sys.exit(2) + click.echo(f"Full traceback:\n{full_traceback}", err=True) + if not status_file: + sys.exit(2) + finally: + # Write status file if requested + if status_file: + try: + with open(status_file, "w", encoding="utf-8") as f: + yaml.dump(status_data, f, default_flow_style=False) + except Exception as status_err: + click.echo(f"Failed to write status file {status_file}: {status_err}", err=True) + sys.exit(4) + + # Exit with error code if operation failed + if not status_data.get("success", False): + sys.exit(3) diff --git a/projects/caliper/cli/s3_import.py b/projects/caliper/cli/s3_import.py index 5f2d8080a..e5d3bc9c9 100644 --- a/projects/caliper/cli/s3_import.py +++ b/projects/caliper/cli/s3_import.py @@ -25,14 +25,13 @@ logger = logging.getLogger(__name__) -def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000) -> list[dict]: +def list_s3_objects(s3_client, bucket: str, prefix: str) -> list[dict]: """List objects in S3 bucket with given prefix. Args: s3_client: Configured boto3 S3 client bucket: S3 bucket name prefix: S3 key prefix to filter objects - max_objects: Maximum number of objects to return Returns: List of S3 object metadata dictionaries @@ -41,9 +40,8 @@ def list_s3_objects(s3_client, bucket: str, prefix: str, max_objects: int = 1000 objects = [] paginator = s3_client.get_paginator("list_objects_v2") - for page in paginator.paginate( - Bucket=bucket, Prefix=prefix, PaginationConfig={"MaxItems": max_objects} - ): + # List ALL objects first - don't limit listing, limit downloads after filtering + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): if "Contents" in page: for obj in page["Contents"]: objects.append( @@ -221,7 +219,7 @@ def run_s3_import_with_explicit_params( Args: bucket: S3 bucket name - prefix: S3 prefix path (full path including s3.prefix/instance/directory) + prefix: S3 prefix path (full path including instance/directory) output_dir: Path to output directory for downloaded files vault: Vault name containing AWS credentials aws_credentials_file: AWS credentials file within vault @@ -273,15 +271,9 @@ def run_s3_import_with_explicit_params( } # List S3 objects (even in dry run mode to show what would be downloaded) - objects = list_s3_objects(s3_client, bucket, import_s3_prefix, max_downloads) + objects = list_s3_objects(s3_client, bucket, import_s3_prefix) logger.info(f"Found {len(objects)} total objects in s3://{bucket}/{import_s3_prefix}") - if objects: - logger.info("Sample object keys:") - for i, obj in enumerate(objects[:5]): # Show first 5 objects - logger.info(f" {i + 1}: {obj['key']}") - if len(objects) > 5: - logger.info(f" ... and {len(objects) - 5} more objects") # Filter objects based on parameters logger.info( @@ -299,6 +291,14 @@ def __init__(self): download_objects = filter_objects_for_download(objects, FilterConfig()) logger.info(f"After filtering: {len(download_objects)} objects match the import filters") + # Sort by last_modified timestamp in descending order (most recent first) + download_objects.sort(key=lambda obj: obj["last_modified"], reverse=True) + + # Apply max_downloads limit after filtering to get the most relevant files + if len(download_objects) > max_downloads: + logger.info(f"Limiting downloads to first {max_downloads} matching files") + download_objects = download_objects[:max_downloads] + if not download_objects: logger.warning("No objects match the import filters") return { diff --git a/projects/caliper/engine/ai_eval.py b/projects/caliper/engine/ai_eval.py index 5a9d6953a..f22cea9ad 100644 --- a/projects/caliper/engine/ai_eval.py +++ b/projects/caliper/engine/ai_eval.py @@ -27,6 +27,12 @@ def run_ai_eval_export( ) # Build base payload from plugin + if not hasattr(plugin, "build_ai_data_payload"): + raise AttributeError( + f"Plugin {plugin_module} does not support AI evaluation export. " + "Missing build_ai_data_payload method." + ) + build = plugin.build_ai_data_payload payload = build(model) @@ -75,7 +81,11 @@ def _export_test_entries_with_artifacts_engine( exported_entries = [] # Get the specific files we want to copy from the plugin - target_files = plugin.get_ai_eval_artifact_files(model) + if hasattr(plugin, "get_ai_eval_artifact_files"): + target_files = plugin.get_ai_eval_artifact_files(model) + else: + # Fallback: copy common test artifacts + target_files = ["*.log", "*.json", "*.yaml", "*.yml"] for idx, record in enumerate(model.unified_result_records): # Create directory for this test entry @@ -93,29 +103,68 @@ def _export_test_entries_with_artifacts_engine( # Copy target files if they exist for target_file in target_files: - source_file = base_dir / target_file - if source_file.exists(): - # Create target directory structure in test entry dir - target_path = test_entry_dir / Path(target_file).name - target_path.parent.mkdir(parents=True, exist_ok=True) - - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "size_bytes": source_file.stat().st_size, - } + # Handle plugin-provided paths directly, expand fallback patterns + if hasattr(plugin, "get_ai_eval_artifact_files"): + # Plugin provided specific files - treat as direct paths + source_file = base_dir / target_file + if source_file.exists(): + # Create target directory structure in test entry dir + target_path = test_entry_dir / Path(target_file).name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(source_file, target_path) + entry_info["copied_files"].append( + { + "source": str(source_file), + "target": str(target_path), + "size_bytes": source_file.stat().st_size, + } + ) + logger.debug(f"Copied {source_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {source_file}: {e}") + entry_info["missing_files"].append( + {"file": str(source_file), "error": str(e)} + ) + else: + entry_info["missing_files"].append( + {"file": str(source_file), "error": "File does not exist"} ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) + # Fallback mode - expand glob patterns + import glob + + pattern_path = base_dir / target_file + matched_files = glob.glob(str(pattern_path)) + + if matched_files: + # Copy all matched files + for matched_file_str in matched_files: + matched_file = Path(matched_file_str) + target_path = test_entry_dir / matched_file.name + target_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(matched_file, target_path) + entry_info["copied_files"].append( + { + "source": str(matched_file), + "target": str(target_path), + "size_bytes": matched_file.stat().st_size, + } + ) + logger.debug(f"Copied {matched_file} -> {target_path}") + except Exception as e: + logger.warning(f"Failed to copy {matched_file}: {e}") + entry_info["missing_files"].append( + {"file": str(matched_file), "error": str(e)} + ) + else: + # Pattern matched no files + entry_info["missing_files"].append( + {"file": str(pattern_path), "error": "Pattern matched no files"} + ) # Write entry metadata entry_metadata_file = test_entry_dir / "entry_metadata.json" diff --git a/projects/caliper/engine/kpi/analyze.py b/projects/caliper/engine/kpi/analyze.py index 8b96ab459..a6430af31 100644 --- a/projects/caliper/engine/kpi/analyze.py +++ b/projects/caliper/engine/kpi/analyze.py @@ -1,295 +1,506 @@ -"""Regression analysis vs baseline KPI set.""" +"""Generic KPI regression analysis against historical baselines.""" from __future__ import annotations import json import logging import time +from dataclasses import dataclass, field from pathlib import Path from typing import Any +import yaml + logger = logging.getLogger(__name__) -def run_kpi_analysis( - current_kpi_file: Path, - historical_data_dir: Path, - output_file: Path, - plugin_module: str, -) -> int: - """Run KPI analysis and generate output file. +@dataclass +class AnalysisConfig: + """Configuration for KPI regression analysis. - Args: - current_kpi_file: Path to current KPI JSON file - historical_data_dir: Directory containing historical KPI files - output_file: Path where analysis results will be written - plugin_module: Plugin module name for analysis + comparison_keys: Label keys that define what we compare against. + Records must differ on at least one comparison key to be distinct baselines. + E.g. ["version"] means we test the current version against other versions. + ignored_keys: Label keys excluded when matching current to baseline records. + E.g. ["os"] means we match across operating systems. + sorting_keys: Label keys used to order entries in the output report. + max_relative_regression: Fraction threshold for flagging regression (0.1 = 10%). + min_baseline_points: Minimum number of baseline data points required to run a test. + """ - Returns: - Exit code (0 for success, non-zero for failure) + comparison_keys: list[str] = field(default_factory=list) + ignored_keys: list[str] = field(default_factory=list) + sorting_keys: list[str] = field(default_factory=list) + max_relative_regression: float = 0.1 + min_baseline_points: int = 1 + + +@dataclass +class KpiTestResult: + """Result of a single KPI regression test.""" + + kpi_id: str + labels: dict[str, Any] + current_value: float + baseline_mean: float + relative_change: float + higher_is_better: bool + regression: bool + baseline_count: int + + +@dataclass +class AnalysisReport: + """Full analysis report.""" + + status: str # "success", "no_regression", "regression_detected", "error" + processed: dict[str, Any] + tested: list[dict[str, Any]] + results: list[dict[str, Any]] + overall: dict[str, Any] + + +def _load_analysis_config(plugin_module: str) -> AnalysisConfig: + """Load analysis config from plugin module if available, else use defaults. + + Plugins can expose an `analysis_config` dict or `get_analysis_config()` callable. """ try: - logger.info(f"Running KPI analysis for: {current_kpi_file}") - logger.info(f"Historical data directory: {historical_data_dir}") - logger.info(f"Output file: {output_file}") - logger.info(f"Plugin module: {plugin_module}") + mod = __import__(plugin_module, fromlist=[""]) + if hasattr(mod, "get_analysis_config"): + raw = mod.get_analysis_config() + elif hasattr(mod, "analysis_config"): + raw = mod.analysis_config + else: + return AnalysisConfig() + + if isinstance(raw, AnalysisConfig): + return raw + if isinstance(raw, dict): + return AnalysisConfig( + **{k: v for k, v in raw.items() if k in AnalysisConfig.__dataclass_fields__} + ) + except (ImportError, Exception) as exc: + logger.debug("Could not load analysis config from plugin %s: %s", plugin_module, exc) + + return AnalysisConfig() + + +def _match_key( + labels: dict[str, Any], ignored_keys: list[str], comparison_keys: list[str] +) -> tuple: + """Build a hashable match key from labels, excluding ignored and comparison keys.""" + excluded = set(ignored_keys) | set(comparison_keys) + return tuple(sorted((k, str(v)) for k, v in labels.items() if k not in excluded)) + + +def _extract_kpi_records_from_hierarchical(data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract flat KPI records from a schema_version=2 hierarchical document.""" + records = [] + for test in data.get("tests", []): + test_labels = test.get("labels", {}) + for kpi in test.get("kpis", []): + records.append( + { + "kpi_id": kpi.get("id"), + "value": kpi.get("value"), + "unit": kpi.get("unit"), + "higher_is_better": kpi.get("higher_is_better", True), + "labels": test_labels, + "run_id": test.get("run_id"), + "metadata": test.get("metadata", {}), + } + ) + return records - # Check if current KPI file exists - if not current_kpi_file.exists(): - logger.error(f"Current KPI file not found: {current_kpi_file}") - return 1 - # Check if historical data directory exists - if not historical_data_dir.exists(): - logger.error(f"Historical data directory not found: {historical_data_dir}") - return 1 - - # Find all historical KPI files - historical_kpi_files = [] - for kpi_file in historical_data_dir.rglob("kpis.json"): - historical_kpi_files.append(str(kpi_file)) - logger.info(f"Found historical KPI file: {kpi_file}") - - if not historical_kpi_files: - logger.warning("No historical KPI files found for analysis") - # Return special exit code 2 to indicate warning (no historical data) - analysis_result = { - "status": "warning", - "message": "no historical KPI found for regression testing", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": 0, - "plugin_module": plugin_module, - "completed_at": time.time(), - } - # Write warning result to output file - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - return 2 # Special exit code for warning - - logger.info(f"Found {len(historical_kpi_files)} historical KPI files for analysis") - - # Create analysis result - analysis_result = { - "status": "success (stub)", - "current_kpi_file": str(current_kpi_file), - "historical_kpi_files": historical_kpi_files, - "baseline_files_count": len(historical_kpi_files), - "plugin_module": plugin_module, - "completed_at": time.time(), - } +def _build_baseline_index( + baseline_kpi_data: dict[Path, dict[str, Any]], + config: AnalysisConfig, +) -> dict[tuple, list[dict[str, Any]]]: + """Index baseline records by (kpi_id, match_key) for fast lookup. - # Create output directory if needed - output_file.parent.mkdir(parents=True, exist_ok=True) + Returns mapping from (kpi_id, match_key) -> list of baseline records. + """ + index: dict[tuple, list[dict[str, Any]]] = {} + for _path, data in baseline_kpi_data.items(): + records = _extract_kpi_records_from_hierarchical(data) + for rec in records: + kpi_id = rec.get("kpi_id") + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + index.setdefault(key, []).append(rec) + return index + + +def _run_regression_test( + current: dict[str, Any], + baselines: list[dict[str, Any]], + config: AnalysisConfig, +) -> KpiTestResult: + """Run a regression test for a single KPI record against its baselines.""" + current_value = float(current["value"]) + higher_is_better = current.get("higher_is_better", True) + baseline_values = [float(b["value"]) for b in baselines if b.get("value") is not None] + baseline_mean = sum(baseline_values) / len(baseline_values) + + if baseline_mean == 0: + relative_change = 0.0 + else: + relative_change = (current_value - baseline_mean) / abs(baseline_mean) - # Write analysis results to output file - with open(output_file, "w") as f: - json.dump(analysis_result, f, indent=2) - f.write("\n") # Add EOL at EOF + # Determine regression: if higher is better, a negative change is regression. + # If lower is better, a positive change is regression. + if higher_is_better: + regression = relative_change < -config.max_relative_regression + else: + regression = relative_change > config.max_relative_regression - logger.info(f"Analysis completed successfully, results written to: {output_file}") - return 0 + return KpiTestResult( + kpi_id=current["kpi_id"], + labels=current.get("labels", {}), + current_value=current_value, + baseline_mean=round(baseline_mean, 6), + relative_change=round(relative_change, 6), + higher_is_better=higher_is_better, + regression=regression, + baseline_count=len(baseline_values), + ) - except Exception as e: - logger.exception("KPI analysis failed") - # Create failure result - failure_result = { - "status": "failed", - "error": str(e), - "current_kpi_file": str(current_kpi_file) if current_kpi_file else None, - "completed_at": time.time(), - } +def _sort_results(results: list[KpiTestResult], sorting_keys: list[str]) -> list[KpiTestResult]: + """Sort results by sorting keys extracted from labels, then by kpi_id.""" - try: - # Attempt to write failure result - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(failure_result, f, indent=2) - f.write("\n") # Add EOL at EOF - except Exception: - logger.exception("Failed to write error result to output file") + def sort_key(r: KpiTestResult): + label_key = tuple(str(r.labels.get(k, "")) for k in sorting_keys) + return (*label_key, r.kpi_id) - return 1 + return sorted(results, key=sort_key) -def analyze_kpis( - postprocess_config, # CaliperOrchestrationPostprocessConfig +def _build_yaml_report( + results: list[KpiTestResult], + config: AnalysisConfig, + current_source: str, + baseline_sources: list[str], + skipped: list[dict[str, Any]], +) -> dict[str, Any]: + """Build the final YAML-serializable report structure.""" + regressions = [r for r in results if r.regression] + + if regressions: + overall_status = "REGRESSION_DETECTED" + else: + overall_status = "PASS" + + report = { + "analysis": { + "status": overall_status, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "config": { + "comparison_keys": config.comparison_keys, + "ignored_keys": config.ignored_keys, + "sorting_keys": config.sorting_keys, + "max_relative_regression": config.max_relative_regression, + "min_baseline_points": config.min_baseline_points, + }, + }, + "processed": { + "current_source": current_source, + "baseline_sources": baseline_sources, + "baseline_source_count": len(baseline_sources), + }, + "tested": { + "total_kpis": len(results), + "regressions": len(regressions), + "passes": len(results) - len(regressions), + "skipped": len(skipped), + }, + "results": [ + { + "kpi_id": r.kpi_id, + "labels": r.labels, + "current_value": r.current_value, + "baseline_mean": r.baseline_mean, + "relative_change_pct": round(r.relative_change * 100, 2), + "higher_is_better": r.higher_is_better, + "verdict": "REGRESSION" if r.regression else "PASS", + "baseline_count": r.baseline_count, + } + for r in results + ], + "overall": { + "verdict": overall_status, + "regression_count": len(regressions), + "total_tested": len(results), + "total_skipped": len(skipped), + }, + } + + if skipped: + report["skipped"] = skipped + + return report + + +def run_kpi_analysis( + current_kpi_file: Path, + historical_data_dir: Path, + output_file: Path, plugin_module: str, - base_dir: Path, - output_dir: Path, - current_kpis_file: Path, ) -> dict[str, Any]: - """Run KPI analysis step and return result status. + """Run KPI regression analysis and generate a YAML report. + + Args: + current_kpi_file: Path to current KPI JSON file (hierarchical schema v2) + historical_data_dir: Directory containing historical KPI files (kpis.json) + output_file: Path where YAML analysis report will be written + plugin_module: Plugin module name (for loading analysis config) - This is the orchestration interface for KPI analysis. + Returns: + Status dictionary with success/error information and exit_code for CLI compatibility """ - if not postprocess_config.analyze.enabled: - return {"status": "disabled", "reason": "analyze disabled"} + try: + logger.info("Running KPI regression analysis") + logger.info(" current_kpi_file: %s", current_kpi_file) + logger.info(" historical_data_dir: %s", historical_data_dir) + logger.info(" output_file: %s", output_file) + logger.info(" plugin_module: %s", plugin_module) + + if not current_kpi_file.exists(): + logger.error("Current KPI file not found: %s", current_kpi_file) + return { + "status": "failed", + "success": False, + "error": f"Current KPI file not found: {current_kpi_file}", + "exit_code": 1, + "completed_at": time.time(), + } - # Determine paths for analysis - analyze_config = postprocess_config.analyze + if not historical_data_dir.exists(): + logger.error("Historical data directory not found: %s", historical_data_dir) + return { + "status": "failed", + "success": False, + "error": f"Historical data directory not found: {historical_data_dir}", + "exit_code": 1, + "completed_at": time.time(), + } - # Historical KPIs directory path - historical_kpis_dir = Path(analyze_config.historical_kpis) - if not historical_kpis_dir.is_absolute(): - historical_kpis_dir = output_dir / historical_kpis_dir + config = _load_analysis_config(plugin_module) + logger.info( + " config: comparison_keys=%s, ignored_keys=%s", + config.comparison_keys, + config.ignored_keys, + ) - # Output path for analysis results - output_path = output_dir / analyze_config.output + # Load current KPIs + with open(current_kpi_file) as f: + current_data = json.load(f) - try: - # Check if required files exist - if not current_kpis_file.exists(): + if current_data.get("schema_version") != "2": + logger.error("Current KPI file must be schema_version 2 (hierarchical)") return { "status": "failed", - "error": f"Current KPI file not found: {current_kpis_file}", + "success": False, + "error": "Current KPI file must be schema_version 2 (hierarchical)", + "exit_code": 1, "completed_at": time.time(), } - if not historical_kpis_dir.exists(): + current_records = _extract_kpi_records_from_hierarchical(current_data) + if not current_records: + logger.warning("No KPI records found in current file") return { "status": "failed", - "error": f"Historical KPIs directory not found: {historical_kpis_dir}", + "success": False, + "error": "No KPI records found in current file", + "exit_code": 1, "completed_at": time.time(), } - # Log command to reproduce this step - from projects.caliper.orchestration.step_logging import log_analyze_command + # Load baseline KPIs + baseline_kpi_data = find_baseline_kpis(historical_data_dir) + if not baseline_kpi_data: + _write_warning_report(output_file, current_kpi_file, plugin_module) + return { + "status": "warning", + "success": True, + "message": "no historical KPI found for regression testing", + "output_file": str(output_file), + "exit_code": 2, + "completed_at": time.time(), + } - log_analyze_command( - base_dir=base_dir, - plugin_module=plugin_module, - current_kpis_path=current_kpis_file, - historical_kpis_dir=historical_kpis_dir, - output_path=output_path, + # Build baseline index + baseline_index = _build_baseline_index(baseline_kpi_data, config) + + # Run regression tests + results: list[KpiTestResult] = [] + skipped: list[dict[str, Any]] = [] + + for rec in current_records: + kpi_id = rec.get("kpi_id") + value = rec.get("value") + + # Skip non-scalar values (2D KPIs, lists, etc.) + if not isinstance(value, (int, float)): + skipped.append({"kpi_id": kpi_id, "reason": "non-scalar value"}) + continue + + labels = rec.get("labels", {}) + mk = _match_key(labels, config.ignored_keys, config.comparison_keys) + key = (kpi_id, mk) + + baselines = baseline_index.get(key, []) + # Filter to only scalar baselines + scalar_baselines = [b for b in baselines if isinstance(b.get("value"), (int, float))] + + if len(scalar_baselines) < config.min_baseline_points: + skipped.append( + { + "kpi_id": kpi_id, + "labels": labels, + "reason": f"insufficient baselines ({len(scalar_baselines)} < {config.min_baseline_points})", + } + ) + continue + + result = _run_regression_test(rec, scalar_baselines, config) + results.append(result) + + # Sort results + results = _sort_results(results, config.sorting_keys) + + # Build report + baseline_sources = [str(p) for p in baseline_kpi_data.keys()] + report = _build_yaml_report( + results=results, + config=config, + current_source=str(current_kpi_file), + baseline_sources=baseline_sources, + skipped=skipped, ) - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=current_kpis_file, - historical_data_dir=historical_kpis_dir, - output_file=output_path, - plugin_module=plugin_module, + # Write YAML report + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + + regressions = report["overall"]["regression_count"] + total = report["overall"]["total_tested"] + logger.info( + "Analysis complete: %d/%d KPIs tested, %d regressions", + total, + len(current_records), + regressions, ) - if exit_code == 0: - # Read the results from the output file - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return success with output file path for notifications - return { - "status": "success", - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "failed", - "error": "Analysis completed but no output file was generated", - "completed_at": time.time(), - } - elif exit_code == 2: - # Warning: no historical data found - if output_path.exists(): - with open(output_path) as f: - result_data = json.load(f) - - # Return warning status with message - return { - "status": "warning", - "message": result_data.get( - "message", "no historical KPI found for regression testing" - ), - "output_file": str(output_path), - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": time.time(), - } - else: - return { - "status": "warning", - "message": "no historical KPI found for regression testing", - "baseline_files_count": 0, - "completed_at": time.time(), - } + if regressions > 0: + return { + "status": "regression_detected", + "success": True, + "regressions_detected": True, + "output_file": str(output_file), + "exit_code": 3, + "completed_at": time.time(), + } else: - # Read error from output file if it exists - error_msg = f"Analysis failed with exit code {exit_code}" - if output_path.exists(): - try: - with open(output_path) as f: - result_data = json.load(f) - error_msg = result_data.get("error", error_msg) - except Exception: - pass - return { - "status": "failed", - "error": error_msg, + "status": "success", + "success": True, + "output_file": str(output_file), + "exit_code": 0, "completed_at": time.time(), } except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - logger.exception(f"Analysis step failed with {error_msg}") + logger.exception("KPI analysis failed") return { "status": "failed", - "error": error_msg, - "error_type": type(e).__name__, + "success": False, + "error": f"KPI analysis failed: {e}", + "exit_code": 1, "completed_at": time.time(), } -def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: - """Load all kpis.json files from historical directory and return a mapping of path to loaded JSON object. +def _write_warning_report(output_file: Path, current_kpi_file: Path, plugin_module: str) -> None: + """Write a warning-level report when no baselines are available.""" + report = { + "analysis": { + "status": "NO_BASELINE", + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }, + "processed": { + "current_source": str(current_kpi_file), + "baseline_sources": [], + "baseline_source_count": 0, + }, + "tested": {"total_kpis": 0, "regressions": 0, "passes": 0, "skipped": 0}, + "results": [], + "overall": { + "verdict": "NO_BASELINE", + "message": "No historical KPI files found for regression testing", + "regression_count": 0, + "total_tested": 0, + "total_skipped": 0, + }, + } + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + yaml.dump(report, f, default_flow_style=False, sort_keys=False, allow_unicode=True) - Args: - historical_dir: Directory to search for historical kpis.json files - Returns: - Dictionary mapping file paths to loaded KPI JSON objects +def find_baseline_kpis(historical_dir: Path) -> dict[Path, dict[str, Any]]: + """Load all kpis.json files from historical directory. + + Returns mapping of file paths to loaded hierarchical KPI data (schema v2 only). """ - baseline_kpis = {} + baseline_kpis: dict[Path, dict[str, Any]] = {} kpi_files = list(historical_dir.rglob("kpis.json")) if not kpi_files: - logger.warning(f"No kpis.json files found in historical directory: {historical_dir}") + logger.warning("No kpis.json files found in: %s", historical_dir) return baseline_kpis - logger.info(f"Found {len(kpi_files)} historical KPI files to load") + logger.info("Found %d historical KPI files to load", len(kpi_files)) for kpi_file in kpi_files: try: with open(kpi_file) as f: kpi_data = json.load(f) - # Validate that it's a hierarchical format (schema_version 2) schema_version = kpi_data.get("schema_version", "unknown") if schema_version != "2": logger.warning( - f"Skipping {kpi_file}: unsupported schema version {schema_version} (only version 2 supported)" + "Skipping %s: unsupported schema version %s", kpi_file, schema_version ) continue baseline_kpis[kpi_file] = kpi_data - logger.debug(f"Loaded KPI file: {kpi_file}") - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON in {kpi_file}: {e}") - continue - except FileNotFoundError as e: - logger.error(f"KPI file not found: {kpi_file}: {e}") - continue + logger.debug("Loaded baseline: %s", kpi_file) + + except json.JSONDecodeError: + # Try parsing as JSONL (v1 format) fallback + try: + v1_records = [] + with open(kpi_file) as f: + for line in f: + if line.strip(): + v1_records.append(json.loads(line)) + + if v1_records: + v2_data = _convert_v1_to_v2(v1_records) + baseline_kpis[kpi_file] = v2_data + logger.debug("Loaded baseline (converted from v1): %s", kpi_file) + except Exception: + logger.error("Failed to parse %s as JSON or JSONL", kpi_file) except Exception as e: - logger.error(f"Failed to load KPI file {kpi_file}: {e}") - continue + logger.error("Failed to load %s: %s", kpi_file, e) - logger.info(f"Successfully loaded {len(baseline_kpis)} historical KPI files") + logger.info("Successfully loaded %d historical KPI files", len(baseline_kpis)) return baseline_kpis @@ -301,52 +512,276 @@ def run_analyze( plugin: Any = None, ) -> dict[str, Any]: """Run KPI analysis against ALL baseline files (CLI interface).""" - # For CLI use, we need to extract the baseline directory from the dict - # This is a bit of a hack, but the CLI interface expects a directory - if baseline_kpis: - # Get the parent directory of the first baseline file - first_baseline_path = next(iter(baseline_kpis.keys())) - historical_dir = first_baseline_path.parent - - # Extract plugin module name if available - plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" - - # Call the core analysis function - exit_code = run_kpi_analysis( - current_kpi_file=Path(current_path), - historical_data_dir=historical_dir, - output_file=Path(output_path), + if not baseline_kpis: + return { + "status": "failed", + "error": "No baseline KPI files provided", + "completed_at": time.time(), + } + + first_baseline_path = next(iter(baseline_kpis.keys())) + historical_dir = first_baseline_path.parent + + plugin_module = getattr(plugin, "__module__", "unknown") if plugin else "unknown" + + result = run_kpi_analysis( + current_kpi_file=Path(current_path), + historical_data_dir=historical_dir, + output_file=Path(output_path), + plugin_module=plugin_module, + ) + + # Extract just the relevant fields for this legacy interface + return { + "status": result.get("status"), + "message": result.get("message"), + "completed_at": result.get("completed_at"), + } + + +def _convert_v1_to_v2(v1_records: list[dict]) -> dict: + """Convert v1 JSONL records to v2 hierarchical format.""" + metrics = {} + for record in v1_records: + # Extract metric name and value + metric_name = record.get("name", "unknown_metric") + value = record.get("value") + unit = record.get("unit") + labels = record.get("labels", {}) + + # Create metric entry in v2 format + metric_entry = { + "value": value, + "unit": unit, + "labels": labels, + } + + # Determine if higher is better from labels or default to false + higher_is_better = labels.get("higher_is_better", False) + metric_entry["higher_is_better"] = higher_is_better + + metrics[metric_name] = metric_entry + + return { + "schema_version": "2", + "metrics": metrics, + "metadata": { + "converted_from_v1": True, + "original_record_count": len(v1_records), + }, + } + + +def ensure_kpi_file_v2_format(file_path: Path) -> tuple[Path, bool]: + """Convert KPI file to v2 hierarchical format if it's in v1 JSONL format. + + Args: + file_path: Path to KPI file (v1 or v2 format) + + Returns: + Tuple of (converted_file_path, is_temporary_file) + - converted_file_path: Path to v2 format file + - is_temporary_file: True if a temporary file was created and needs cleanup + """ + import json + import tempfile + + logger.debug("Checking format of KPI file: %s", file_path) + + try: + with open(file_path) as f: + content = f.read().strip() + if not content: + raise ValueError(f"KPI file {file_path} is empty") + + # Try to parse as v2 (hierarchical JSON) first + try: + data = json.loads(content) + if isinstance(data, dict) and "schema_version" in data: + # Already in v2 format + logger.debug("KPI file %s is already in v2 format", file_path) + return file_path, False + elif isinstance(data, list): + raise ValueError(f"KPI file {file_path} appears to be in unexpected list format") + else: + # Single dict but missing schema_version - might be v1 single record + logger.debug( + "KPI file %s appears to be single record without schema_version", file_path + ) + except json.JSONDecodeError: + # Not valid single JSON, might be JSONL (v1 format) + logger.debug("KPI file %s not valid single JSON, checking JSONL format", file_path) + + # Try to parse as v1 (JSONL) format + lines = content.split("\n") + non_empty_lines = [line.strip() for line in lines if line.strip()] + + if len(non_empty_lines) == 0: + raise ValueError(f"KPI file {file_path} contains no data") + + # Try to parse first line to validate v1 format + try: + first_record = json.loads(non_empty_lines[0]) + if not isinstance(first_record, dict): + raise ValueError(f"KPI file {file_path} first line is not a JSON object") + + # Parse all lines as v1 JSONL records + v1_records = [] + for i, line in enumerate(non_empty_lines): + try: + record = json.loads(line) + v1_records.append(record) + except json.JSONDecodeError as line_err: + raise ValueError( + f"KPI file {file_path} line {i + 1} has invalid JSON: {line_err}" + ) from line_err + + # Convert v1 records to v2 hierarchical format + logger.info( + "Converting KPI file %s from v1 (JSONL) to v2 (hierarchical) format", file_path + ) + + # Transform to v2 + v2_data = _convert_v1_to_v2(v1_records) + + # Write v2 format to temporary file + temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + try: + json.dump(v2_data, temp_file, indent=2, ensure_ascii=False) + temp_file.flush() + temp_path = Path(temp_file.name) + logger.info( + "Converted %s (%d records) to v2 format at %s", + file_path, + len(v1_records), + temp_path, + ) + return temp_path, True + except Exception as write_err: + temp_file.close() + Path(temp_file.name).unlink(missing_ok=True) + raise ValueError( + f"Failed to write converted v2 format file: {write_err}" + ) from write_err + finally: + temp_file.close() + + except Exception as v1_err: + raise ValueError( + f"KPI file {file_path} is not in valid v1 (JSONL) or v2 (hierarchical) format: {v1_err}" + ) from v1_err + + except Exception as format_err: + raise ValueError( + f"Failed to process KPI file format for {file_path}: {format_err}" + ) from format_err + + +def analyze_kpis( + current_kpis_file: Path, + historical_kpis_dir: Path, + output_file: Path, + plugin_module: str, +) -> dict[str, Any]: + """Analyze KPIs with automatic v1/v2 format conversion. + + This function handles: + 1. Format detection and conversion from v1 (JSONL) to v2 (hierarchical) + 2. Plugin loading + 3. Analysis execution + 4. Temporary file cleanup + 5. Result formatting + + Args: + current_kpis_file: Path to current KPIs file (v1 or v2 format) + historical_kpis_dir: Directory containing historical KPI files + output_file: Path where analysis results will be written + plugin_module: Plugin module name for KPI definitions and analysis rules + + Returns: + Dictionary with status information: + - success: bool - whether analysis completed successfully + - output_file: str - path to analysis results (if successful) + - error: str - error message (if failed) + """ + + try: + # Use directory-based analysis (supports multiple baseline files) + result = run_kpi_analysis( + current_kpi_file=current_kpis_file, + historical_data_dir=historical_kpis_dir, + output_file=output_file, plugin_module=plugin_module, ) - if exit_code == 0: - # Read results and convert to expected format for CLI - try: - with open(Path(output_path)) as f: - result_data = json.load(f) - return { - "status": "success", - "baseline_files_count": result_data.get("baseline_files_count", 0), - "completed_at": result_data.get("completed_at", time.time()), - } - except Exception as e: - return { - "status": "failed", - "error": f"Failed to read analysis results: {e}", - "completed_at": time.time(), - } + # Convert to the expected format for orchestration and CLI + if result.get("status") == "success": + return { + "status": "success", + "success": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "warning": + return { + "status": "warning", + "success": True, # Warning is still success + "message": result.get("message"), + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } + elif result.get("status") == "regression_detected": + return { + "status": "regression_detected", + "success": True, # Regression detection is successful analysis + "regressions_detected": True, + "output_file": result.get("output_file"), + "completed_at": result.get("completed_at"), + } else: return { "status": "failed", - "error": f"Analysis failed with exit code {exit_code}", - "completed_at": time.time(), + "success": False, + "error": result.get("error", "Analysis failed"), + "completed_at": result.get("completed_at"), } - else: + + except Exception as e: + logger.exception("KPI analysis with format conversion failed") return { - "status": "failed", - "error": "No baseline KPI files provided", + "success": False, + "error": str(e), "completed_at": time.time(), } +def status_dict_to_exit_code(status_dict: dict[str, Any]) -> int: + """Convert a status dictionary to an exit code for CLI use. + + Args: + status_dict: Status dictionary from engine analysis functions + + Returns: + Exit code: 0=success, 1=error, 2=warning (no baseline), 3=regression detected + """ + # If the status dict includes an explicit exit_code, use it + if "exit_code" in status_dict: + return status_dict["exit_code"] + + # Otherwise, determine exit code from status fields + if not status_dict.get("success", False): + return 1 # General error + + status = status_dict.get("status", "") + + if status == "success": + return 0 + elif status == "warning" or "no baseline" in status_dict.get("message", "").lower(): + return 2 # Warning (no baseline data) + elif status == "regression_detected" or status_dict.get("regressions_detected"): + return 3 # Regression detected + else: + return 0 # Default to success if unclear + + # EOF diff --git a/projects/caliper/engine/kpi/analyze_hierarchical.py b/projects/caliper/engine/kpi/analyze_hierarchical.py new file mode 100644 index 000000000..88d56ee92 --- /dev/null +++ b/projects/caliper/engine/kpi/analyze_hierarchical.py @@ -0,0 +1,313 @@ +"""KPI analysis for hierarchical format.""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def _load_kpis_any_format(kpi_file_path: Path) -> dict[str, Any]: + """Load KPIs from file, handling both v1 (JSONL) and v2 (hierarchical) formats. + + Returns: + Dictionary in hierarchical format with schema_version and metrics + """ + with open(kpi_file_path) as f: + content = f.read().strip() + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Already hierarchical format + return data + else: + # Unknown JSON format, convert list to v1 JSONL handling + raise ValueError("Unknown JSON format") + + except (json.JSONDecodeError, ValueError): + # Try to parse as JSONL (v1 format) + kpis = [] + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + # Convert v1 format to hierarchical format + metrics = {} + for kpi in kpis: + kpi_id = kpi.get("kpi_id") + if kpi_id: + value = kpi.get("value") + labels = kpi.get("labels", {}) + higher_is_better = labels.get("higher_is_better", False) + + metrics[kpi_id] = { + "value": value, + "higher_is_better": higher_is_better, + "labels": labels, + "unit": kpi.get("unit"), + "run_id": kpi.get("run_id"), + "timestamp": kpi.get("timestamp"), + } + + return {"schema_version": "2", "metrics": metrics, "converted_from": "v1_jsonl"} + + +def analyze_hierarchical_kpis( + current_kpis_path: Path, + baseline_kpis_path: Path, + output_path: Path, + plugin: Any = None, +) -> dict[str, Any]: + """ + Analyze hierarchical KPI format for regressions. + + Args: + current_kpis_path: Path to current KPIs JSON file (hierarchical format) + baseline_kpis_path: Path to baseline KPIs JSON file (hierarchical format) + output_path: Path where analysis results will be written + plugin: Caliper plugin instance for KPI definitions and analysis rules + + Returns: + Analysis result dictionary with status, findings, etc. + """ + try: + # Load KPI files (supports both v1 JSONL and v2 hierarchical formats) + current_data = _load_kpis_any_format(current_kpis_path) + baseline_data = _load_kpis_any_format(baseline_kpis_path) + + # Both files are now guaranteed to be in hierarchical format with schema_version 2 + logger.info(f"Loaded current KPIs: {current_data.get('converted_from', 'v2_hierarchical')}") + logger.info( + f"Loaded baseline KPIs: {baseline_data.get('converted_from', 'v2_hierarchical')}" + ) + + # Analyze metrics using plugin-aware logic + current_metrics = current_data.get("metrics", {}) + baseline_metrics = baseline_data.get("metrics", {}) + + findings = [] + regressions = 0 + improvements = 0 + + # Get plugin-specific KPI definitions if available + if plugin and hasattr(plugin, "compute_kpis"): + try: + # Try to get KPI metadata from plugin + # This is a stub - plugins may expose KPI definitions differently + logger.debug(f"Plugin available for analysis: {plugin.__class__.__name__}") + except Exception as e: + logger.warning(f"Could not get KPI definitions from plugin: {e}") + + # Compare common metrics + for metric_name in current_metrics: + if metric_name in baseline_metrics: + current_metric = current_metrics[metric_name] + baseline_metric = baseline_metrics[metric_name] + + current_value = current_metric.get("value") + baseline_value = baseline_metric.get("value") + + if current_value is not None and baseline_value is not None: + try: + curr_val = float(current_value) + base_val = float(baseline_value) + + # Get direction preference from metric metadata or use default + higher_is_better = current_metric.get("higher_is_better", False) + if "labels" in current_metric: + higher_is_better = current_metric["labels"].get( + "higher_is_better", False + ) + + # Calculate change + change_percent = ( + ((curr_val - base_val) / base_val) * 100 + if base_val != 0 + else float("inf") + ) + + # Default threshold: 5% change (can be made configurable via plugin) + threshold_percent = 5.0 + + # Determine if this is a regression based on direction + is_regression = False + is_improvement = False + + if abs(change_percent) > threshold_percent: + if higher_is_better: + # For metrics where higher is better + if curr_val < base_val: # Decreased + is_regression = True + elif curr_val > base_val: # Increased + is_improvement = True + else: + # For metrics where lower is better + if curr_val > base_val: # Increased + is_regression = True + elif curr_val < base_val: # Decreased + is_improvement = True + + if is_regression: + regressions += 1 + findings.append( + { + "metric": metric_name, + "type": "regression", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + elif is_improvement: + improvements += 1 + findings.append( + { + "metric": metric_name, + "type": "improvement", + "current_value": curr_val, + "baseline_value": base_val, + "change_percent": change_percent, + "higher_is_better": higher_is_better, + "threshold": threshold_percent, + } + ) + + except (TypeError, ValueError): + logger.warning( + f"Could not compare metric {metric_name}: non-numeric values" + ) + + # Create analysis results + analysis_result = { + "analysis_timestamp": time.time(), + "current_file": str(current_kpis_path), + "baseline_file": str(baseline_kpis_path), + "schema_version": "2", + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "findings": findings, + "summary": f"Found {regressions} regressions and {improvements} improvements across {len([m for m in current_metrics if m in baseline_metrics])} metrics", + } + + # Write results + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(analysis_result, f, indent=2) + f.write("\n") + + metrics_tested = len([m for m in current_metrics if m in baseline_metrics]) + logger.info( + f"Analysis completed: {metrics_tested} verifications, {regressions} regressions, {improvements} improvements" + ) + + return { + "status": "success", + "findings_count": len(findings), + "regressions_count": regressions, + "improvements_count": improvements, + "output_file": str(output_path), + "metrics_compared": len([m for m in current_metrics if m in baseline_metrics]), + "completed_at": time.time(), + } + + except FileNotFoundError as e: + return { + "status": "failed", + "error": f"File not found: {e}", + "completed_at": time.time(), + } + except json.JSONDecodeError as e: + return { + "status": "failed", + "error": f"Invalid JSON format: {e}", + "completed_at": time.time(), + } + except Exception as e: + logger.exception("Analysis failed") + return { + "status": "failed", + "error": f"Analysis failed: {e}", + "completed_at": time.time(), + } + + +def find_most_recent_baseline(historical_dir: Path) -> Path | None: + """Find the most recently modified kpis.json file in historical directory.""" + kpi_files = list(historical_dir.rglob("kpis.json")) + if not kpi_files: + logger.warning(f"No kpis.json files found in: {historical_dir}") + return None + + logger.info(f"Discovered {len(kpi_files)} baseline candidate files:") + + # Check format of each file and log results + v1_files = [] + v2_files = [] + failed_files = [] + + for kpi_file in kpi_files: + try: + with open(kpi_file) as f: + content = f.read().strip() + + try: + data = json.loads(content) + if isinstance(data, dict) and data.get("schema_version") == "2": + v2_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v2_hierarchical") + else: + # Try parsing as JSONL (v1) + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (unknown format)" + ) + except (json.JSONDecodeError, ValueError): + # Try as JSONL + try: + lines = [line.strip() for line in content.splitlines() if line.strip()] + if lines and all(json.loads(line) for line in lines): + v1_files.append(kpi_file) + logger.info(f" {kpi_file.relative_to(historical_dir)} → v1_jsonl") + else: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (invalid JSONL)" + ) + except Exception: + failed_files.append(kpi_file) + logger.warning( + f" {kpi_file.relative_to(historical_dir)} → FAILED (parse error)" + ) + except Exception as e: + failed_files.append(kpi_file) + logger.error(f" {kpi_file.relative_to(historical_dir)} → FAILED ({e})") + + logger.info( + f"Summary: {len(v2_files)} v2_hierarchical, {len(v1_files)} v1_jsonl, {len(failed_files)} failed" + ) + + most_recent = max(kpi_files, key=lambda p: p.stat().st_mtime) + logger.info(f"Selected most recent: {most_recent.relative_to(historical_dir)}") + + return most_recent diff --git a/projects/caliper/engine/kpi/csv_export.py b/projects/caliper/engine/kpi/csv_export.py new file mode 100644 index 000000000..baf9ef032 --- /dev/null +++ b/projects/caliper/engine/kpi/csv_export.py @@ -0,0 +1,43 @@ +"""Export KPIs to CSV format using plugin interface.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +def export_kpis_to_csv( + *, + plugin: object, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, +) -> str: + """ + Export KPI records to CSV format using the plugin's CSV export method. + + Args: + plugin: PostProcessingPlugin instance with export_kpis_to_csv method + kpi_records: List of KPI record dictionaries + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file as string + + Raises: + AttributeError: If plugin doesn't have export_kpis_to_csv method + """ + if not hasattr(plugin, "export_kpis_to_csv"): + raise AttributeError( + f"Plugin {plugin.__class__.__name__} does not implement export_kpis_to_csv method" + ) + + # Delegate to plugin-specific CSV export implementation + result_path = plugin.export_kpis_to_csv( + kpi_records=kpi_records, + output_path=output_path, + include_header_comments=include_header_comments, + ) + + return result_path diff --git a/projects/caliper/engine/kpi/format.py b/projects/caliper/engine/kpi/format.py new file mode 100644 index 000000000..009cb0e94 --- /dev/null +++ b/projects/caliper/engine/kpi/format.py @@ -0,0 +1,340 @@ +"""KPI output format transformations and utilities.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path +from typing import Any + + +def transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: + """ + Transform flat KPI list into hierarchical JSON structure. + + Groups KPIs by test (run_id), extracting common labels and organizing + KPI metadata (name, help, unit, etc.) for improved readability. + + Args: + kpis: List of flat KPI records from compute_kpis + model: Unified model for accessing plugin metadata + + Returns: + Hierarchical JSON structure organized by test + """ + + if not kpis: + return {"schema_version": "2", "tests": []} + + # Group KPIs by test (run_id) + tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) + + # Get KPI function metadata from the plugin module + from projects.caliper.engine.kpi.decorators import get_kpi_functions + + try: + plugin_module_obj = __import__(model.plugin_module, fromlist=[""]) + kpi_functions = get_kpi_functions(plugin_module_obj) + except (ImportError, AttributeError): + kpi_functions = {} + + # First pass: determine which labels vary across KPIs in the same run + run_label_values: dict[str, dict[str, set]] = defaultdict(lambda: defaultdict(set)) + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + for k, v in kpi.get("labels", {}).items(): + if k == "higher_is_better": + continue + run_label_values[run_id][k].add(str(v)) + + per_kpi_label_keys: dict[str, set[str]] = {} + for run_id, label_vals in run_label_values.items(): + per_kpi_label_keys[run_id] = {k for k, vals in label_vals.items() if len(vals) > 1} + + for kpi in kpis: + run_id = kpi.get("run_id", "unknown") + test_data = tests_data[run_id] + varying_keys = per_kpi_label_keys.get(run_id, set()) + + kpi_labels = kpi.get("labels", {}) + test_labels = { + k: v + for k, v in kpi_labels.items() + if k not in ("higher_is_better",) and k not in varying_keys + } + + if not test_data["labels"]: + test_data["labels"] = test_labels + + # Store test metadata from first KPI + if not test_data["metadata"]: + test_data["metadata"] = { + "timestamp": kpi.get("timestamp"), + "source": kpi.get("source", {}), + "run_id": run_id, + } + + # Per-KPI labels that vary across rate points + kpi_specific_labels = {k: kpi_labels[k] for k in varying_keys if k in kpi_labels} + + # Create KPI record with metadata + kpi_id = kpi.get("kpi_id") + raw_value = kpi.get("value") + + # Resolve _kpi_is_2d first as source of truth when metadata exists + is_2d = False + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + is_2d = getattr(func, "_kpi_is_2d", False) + else: + # Consistent fallback heuristic when metadata is absent + is_2d = bool( + isinstance(raw_value, list) + and raw_value + and isinstance(raw_value[0], (list, tuple)) + and len(raw_value[0]) == 2 + ) + + # Apply tuple-pair structural transform only for confirmed 2D KPIs + if is_2d and isinstance(raw_value, list) and raw_value and len(raw_value) > 0: + first_item = raw_value[0] + if isinstance(first_item, (list, tuple)) and len(first_item) == 2: + try: + # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format + structured_value = { + "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], + "count": len(raw_value), + } + final_value = structured_value + except (ValueError, TypeError, IndexError): + # If conversion fails, preserve original list-of-tuples representation + final_value = raw_value + else: + # Preserve original list-of-tuples representation + final_value = raw_value + else: + # Not 2D or no valid tuple-pair structure, keep original value + final_value = raw_value + + kpi_record: dict[str, Any] = { + "id": kpi_id, + "value": final_value, + "unit": kpi.get("unit"), + "higher_is_better": kpi_labels.get("higher_is_better", True), + "is_2d": is_2d, + } + if kpi_specific_labels: + kpi_record["labels"] = kpi_specific_labels + + # Add KPI metadata from function decorator if available + if kpi_id in kpi_functions: + func = kpi_functions[kpi_id] + kpi_record.update( + { + "name": ( + func.__doc__.replace(" KPI.", "") + if func.__doc__ + else kpi_id.replace("_", " ").title() + ), + "help": getattr(func, "_kpi_help", ""), + } + ) + + # Add 2D-specific metadata if this is a 2D KPI + if is_2d: + kpi_record.update( + { + "x_unit": getattr(func, "_kpi_x_unit", ""), + "x_help": getattr(func, "_kpi_x_help", ""), + "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], + "y_help": getattr(func, "_kpi_y_help", None) + or getattr(func, "_kpi_help", ""), + } + ) + + # Add formatting info if available + if hasattr(func, "_kpi_format"): + kpi_record["format"] = func._kpi_format + else: + # Fallback if no function metadata available + kpi_record.update( + { + "name": kpi_id.replace("_", " ").title(), + "help": f"KPI: {kpi_id}", + } + ) + + test_data["kpis"].append(kpi_record) + + # Convert to final structure + tests_list = [] + for run_id, test_data in tests_data.items(): + tests_list.append( + { + "run_id": run_id, + "labels": test_data["labels"], + "metadata": test_data["metadata"], + "kpis": test_data["kpis"], + } + ) + + return {"schema_version": "2", "tests": tests_list} + + +def write_kpis_in_format( + kpis: list[dict], output_file: Path, format_type: str = "hierarchical", model: Any = None +) -> None: + """ + Write KPIs to file in the specified format. + + Args: + kpis: List of KPI records + output_file: Path to output file + format_type: Format type - "hierarchical" (default) or "jsonl" + model: Unified model (required for hierarchical format) + """ + output_file.parent.mkdir(parents=True, exist_ok=True) + + if format_type == "hierarchical": + if model is None: + raise ValueError("Model is required for hierarchical format") + + # Transform to hierarchical format (schema v2) + hierarchical_data = transform_kpis_to_hierarchical_format(kpis, model) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(hierarchical_data, f, indent=2, ensure_ascii=False) + # Add EOL at EOF if we have data + if kpis: + f.write("\n") + + elif format_type == "jsonl": + # Write as JSONL (schema v1) - matches original behavior + text = "\n".join(json.dumps(kpi, ensure_ascii=False) for kpi in kpis) + ( + "\n" if kpis else "" + ) + with open(output_file, "w", encoding="utf-8") as f: + f.write(text) + + else: + raise ValueError(f"Unknown format type: {format_type}. Use 'hierarchical' or 'jsonl'") + + +def _convert_to_schema_v1_value(raw_value: Any) -> Any: + """ + Convert structured KPI value back to schema-v1 list-of-pairs representation. + + Args: + raw_value: The KPI value, either scalar or structured with data_points/count + + Returns: + Converted value - list of pairs for 2D data, scalar values unchanged + """ + if isinstance(raw_value, dict): + # Check if this is a structured value with data_points + if "data_points" in raw_value and isinstance(raw_value["data_points"], list): + # Convert data_points list back to list-of-pairs format + data_points = raw_value["data_points"] + return [ + [point.get("x"), point.get("y")] + for point in data_points + if isinstance(point, dict) and "x" in point and "y" in point + ] + # For other dict structures, return as-is (preserve existing format) + return raw_value + else: + # Preserve scalar values unchanged + return raw_value + + +def read_kpis_from_file(file_path: Path) -> list[dict]: + """ + Read KPIs from a file, handling both JSONL and hierarchical JSON formats. + + Args: + file_path: Path to the KPI file + + Returns: + List of KPI records in flat format + """ + kpis = [] + + with open(file_path, encoding="utf-8") as f: + content = f.read().strip() + + if not content: + return kpis + + try: + # Try to parse as JSON (hierarchical format) + data = json.loads(content) + + if isinstance(data, dict) and data.get("schema_version") == "2": + # Hierarchical format - flatten it + # Accumulate records in temporary list to ensure atomicity + temp_kpis = [] + + for test in data.get("tests", []): + test_metadata = test.get("metadata", {}) + test_labels = test.get("labels", {}) + + for kpi_record in test.get("kpis", []): + # Convert structured value to schema-v1 format if needed + raw_value = kpi_record.get("value") + converted_value = _convert_to_schema_v1_value(raw_value) + + # Merge test-level labels with any per-KPI labels + merged_labels = { + **test_labels, + **kpi_record.get("labels", {}), + "higher_is_better": kpi_record.get("higher_is_better", True), + } + + run_id = test_metadata.get("run_id") + + flat_kpi = { + "schema_version": "1", + "kpi_id": kpi_record.get("id"), + "value": converted_value, + "unit": kpi_record.get("unit"), + "run_id": run_id, + "run_path": run_id, + "timestamp": test_metadata.get("timestamp"), + "labels": merged_labels, + "source": test_metadata.get("source", {}), + } + + # Preserve schema-v2 metadata fields if present + metadata_fields = [ + "is_2d", + "name", + "help", + "x_unit", + "y_unit", + "x_help", + "y_help", + "format", + ] + for field in metadata_fields: + if field in kpi_record: + flat_kpi[field] = kpi_record[field] + temp_kpis.append(flat_kpi) + + # Only merge into main kpis list after entire branch succeeds + kpis.extend(temp_kpis) + else: + # Unknown JSON format + raise ValueError("Unknown JSON format") + + except json.JSONDecodeError: + # Try to parse as JSONL + for line in content.splitlines(): + line = line.strip() + if line: + try: + kpi = json.loads(line) + kpis.append(kpi) + except json.JSONDecodeError: + continue # Skip invalid lines + + return kpis diff --git a/projects/caliper/engine/kpi/generate.py b/projects/caliper/engine/kpi/generate.py index 27dd575c2..99abacada 100644 --- a/projects/caliper/engine/kpi/generate.py +++ b/projects/caliper/engine/kpi/generate.py @@ -2,10 +2,10 @@ from __future__ import annotations -import json from pathlib import Path from typing import Any +from projects.caliper.engine.kpi.format import write_kpis_in_format from projects.caliper.engine.parse import run_parse from projects.caliper.engine.validation import load_schema, schema_path, validate_instance @@ -18,7 +18,22 @@ def run_kpi_generate( output: Path | None, use_cache: bool, cache_path: Path | None, + format_type: str = "hierarchical", ) -> list[dict[str, Any]]: + """Generate KPI output in specified format. + + Args: + base_dir: Directory containing test artifacts + plugin_module: Name of the plugin module + plugin: Plugin instance + output: Path to write output file + use_cache: Whether to use cached parse results + cache_path: Path to cache file (not used currently) + format_type: Output format - "hierarchical" (default) or "jsonl" + + Returns: + List of KPI records + """ model = run_parse( base_dir=base_dir, plugin_module=plugin_module, @@ -30,7 +45,8 @@ def run_kpi_generate( kpi_schema = load_schema(schema_path("kpi_record.schema.json")) for row in rows: validate_instance(row, kpi_schema, "KPI record") - text = "\n".join(json.dumps(r, ensure_ascii=False) for r in rows) + ("\n" if rows else "") + if output: - output.write_text(text, encoding="utf-8") + write_kpis_in_format(rows, output, format_type, model) + return rows diff --git a/projects/caliper/engine/kpi/metrics_from_kpis.py b/projects/caliper/engine/kpi/metrics_from_kpis.py new file mode 100644 index 000000000..0ba22f833 --- /dev/null +++ b/projects/caliper/engine/kpi/metrics_from_kpis.py @@ -0,0 +1,144 @@ +"""Generic kpis.json -> metrics.json + parameters.json conversion. + +Reads a hierarchical kpis.json (schema v2) and writes per-test-run +metrics.json and parameters.json files into the matching artifact tree +directories. The MLflow export backend picks these up automatically via +``_log_metrics_and_params_from_tree``. + +This replaces project-specific metrics.json generation (e.g. in +mcp_gateway parsers) with a single generic caliper mechanism that works +for every project producing a kpis.json. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +METRICS_FILE = "metrics.json" +PARAMETERS_FILE = "parameters.json" +TEST_LABELS_MARKER = "__test_labels__.yaml" + + +def _build_run_dir_index(artifact_tree: Path) -> dict[str, Path]: + """Map run directory names to their paths using __test_labels__.yaml markers.""" + index: dict[str, Path] = {} + for marker in sorted(artifact_tree.rglob(TEST_LABELS_MARKER)): + if marker.is_file(): + run_dir = marker.parent + try: + rel = run_dir.relative_to(artifact_tree) + except ValueError: + rel = Path(run_dir.name) + index[str(rel)] = run_dir + index[run_dir.name] = run_dir + return index + + +def _is_scalar(value: Any) -> bool: + """Check if a KPI value is a scalar number (not 2D data).""" + return isinstance(value, int | float) and not isinstance(value, bool) + + +def _write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + + +def generate_metrics_from_kpis( + kpis_json_path: Path, + artifact_tree: Path, +) -> dict[str, Any]: + """Convert kpis.json into per-run metrics.json and parameters.json files. + + For each test entry in kpis.json, finds the matching directory under + ``artifact_tree`` (via ``__test_labels__.yaml`` markers) and writes: + + - ``metrics.json``: ``{kpi_id: value}`` for all scalar KPIs + - ``parameters.json``: test-level labels as string key-value pairs + + Args: + kpis_json_path: Path to the kpis.json file (schema v2). + artifact_tree: Root of the caliper artifact tree containing + test run directories with ``__test_labels__.yaml`` markers. + + Returns: + Status dict with counts and any warnings. + """ + if not kpis_json_path.is_file(): + return {"status": "skipped", "reason": f"kpis.json not found: {kpis_json_path}"} + + try: + with kpis_json_path.open(encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Failed to read kpis.json at %s: %s", kpis_json_path, e) + return {"status": "failed", "error": str(e)} + + if not isinstance(data, dict) or data.get("schema_version") != "2": + return {"status": "skipped", "reason": "Not a schema v2 kpis.json"} + + tests = data.get("tests", []) + if not tests: + return {"status": "skipped", "reason": "No tests in kpis.json"} + + run_dir_index = _build_run_dir_index(artifact_tree) + if not run_dir_index: + logger.warning("No test run directories found under %s", artifact_tree) + return {"status": "skipped", "reason": "No run directories with __test_labels__.yaml found"} + + written = 0 + warnings: list[str] = [] + + for test_entry in tests: + run_id = test_entry.get("run_id", "") + test_base_path = test_entry.get("metadata", {}).get("source", {}).get( + "test_base_path", run_id + ) + + run_dir = run_dir_index.get(test_base_path) or run_dir_index.get(run_id) + if run_dir is None: + warnings.append(f"No matching directory for run_id={run_id!r}") + continue + + kpis = test_entry.get("kpis", []) + metrics: dict[str, float] = {} + for kpi in kpis: + kpi_id = kpi.get("id", "") + value = kpi.get("value") + if kpi_id and _is_scalar(value): + metrics[kpi_id] = value + + if metrics: + _write_json(run_dir / METRICS_FILE, metrics) + + labels = test_entry.get("labels", {}) + if labels: + params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} + _write_json(run_dir / PARAMETERS_FILE, params) + + written += 1 + + result: dict[str, Any] = { + "status": "success", + "tests_processed": written, + "total_tests": len(tests), + } + if warnings: + result["warnings"] = warnings + for w in warnings: + logger.warning("kpis-to-metrics: %s", w) + + logger.info( + "Generated metrics.json for %d/%d test(s) from %s", + written, + len(tests), + kpis_json_path.name, + ) + return result diff --git a/projects/caliper/engine/model.py b/projects/caliper/engine/model.py index 3a786f69c..10fc42b04 100644 --- a/projects/caliper/engine/model.py +++ b/projects/caliper/engine/model.py @@ -105,6 +105,52 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Return canonical-shaped KPI dicts (pre-validated).""" return [] + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + # Default implementation: basic CSV export + import csv + + # Ensure parent directory exists before any file operations + output_path.parent.mkdir(parents=True, exist_ok=True) + + if not kpi_records: + output_path.touch() # Create empty file + return str(output_path) + + with open(output_path, "w", newline="", encoding="utf-8") as csvfile: + if include_header_comments: + csvfile.write("# KPI Export\n") + csvfile.write("# Generated by Caliper\n") + + if kpi_records: + # Get all possible fieldnames from all records + fieldnames = set() + for record in kpi_records: + fieldnames.update(record.keys()) + fieldnames = sorted(fieldnames) + + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for record in kpi_records: + writer.writerow(record) + + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Structured JSON for AI agent evaluation.""" return {"schema_version": "1", "run_id": "", "metrics": {}} diff --git a/projects/caliper/engine/parameter_matrix.py b/projects/caliper/engine/parameter_matrix.py index 69c95d3ee..03e720e10 100644 --- a/projects/caliper/engine/parameter_matrix.py +++ b/projects/caliper/engine/parameter_matrix.py @@ -237,8 +237,4 @@ def create_legend_name( legend_name = ", ".join(param_pairs) - # Truncate if too long - if len(legend_name) > max_length: - legend_name = legend_name[: max_length - 3] + "..." - return legend_name diff --git a/projects/caliper/engine/parse.py b/projects/caliper/engine/parse.py index dc0e6cce8..368f712cb 100644 --- a/projects/caliper/engine/parse.py +++ b/projects/caliper/engine/parse.py @@ -2,7 +2,10 @@ from __future__ import annotations +import logging +import time from pathlib import Path +from typing import Any from projects.caliper.engine.cache import ( cache_path_for_test_base, @@ -18,6 +21,8 @@ ) from projects.caliper.engine.traverse import discover_test_bases +logger = logging.getLogger(__name__) + # Validation functions no longer needed - using per-test-base caching @@ -29,6 +34,7 @@ def run_parse( use_cache: bool, force_report_partial: bool = True, show_parameter_matrix: bool = True, + label_filter: dict[str, Any] | None = None, ) -> UnifiedRunModel: """ Run full parse or load valid cache. @@ -38,16 +44,22 @@ def run_parse( base_dir = base_dir.resolve() # Discover test bases - nodes = discover_test_bases(base_dir) + nodes = discover_test_bases(base_dir, label_filter=label_filter) # Always use per-test-base caching all_records = [] cache_refs = [] all_warnings = [] + # Track timing statistics + parse_start_time = time.time() + cached_count = 0 + parsed_count = 0 + parse_fn = plugin.parse for node in nodes: + start_time = time.time() test_base_dir = node.directory cache_file = cache_path_for_test_base(test_base_dir, plugin_module) fp = fingerprint_test_base(test_base_dir, plugin_module) @@ -68,6 +80,10 @@ def run_parse( # Use cached records all_records.extend(cached_records) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + cached_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed from cache in {elapsed_time:.3f}s: {relative_path}") else: # Parse this test base result = parse_fn([node]) # Parse just this node @@ -85,6 +101,16 @@ def run_parse( fingerprint=fp, ) cache_refs.append(str(cache_file)) + elapsed_time = time.time() - start_time + parsed_count += 1 + relative_path = test_base_dir.relative_to(base_dir) + logger.info(f"⏱️ Directory parsed fresh in {elapsed_time:.3f}s: {relative_path}") + + # Log timing summary + total_parse_time = time.time() - parse_start_time + logger.info( + f"⏱️ Parse timing summary: {total_parse_time:.3f}s total ({cached_count} cached, {parsed_count} parsed)" + ) # Create unified model with all records cache_ref_summary = f"per-test-base: {len(cache_refs)} cache files" @@ -98,14 +124,14 @@ def run_parse( if all_warnings and force_report_partial: for w in all_warnings: - print(f"[parse warning] {w}") # noqa: T201 — CLI feedback + logger.warning(f"[parse warning] {w}") # Show parameter matrix if requested if show_parameter_matrix: - print() # noqa: T201 — CLI feedback + logger.info("") matrix_analysis = analyze_parameter_matrix(all_records) matrix_summary = format_parameter_matrix_summary(matrix_analysis) - print(matrix_summary) # noqa: T201 — CLI feedback - print() # noqa: T201 — CLI feedback + logger.info(matrix_summary) + logger.info("") return model diff --git a/projects/caliper/engine/traverse.py b/projects/caliper/engine/traverse.py index 461e4f7d6..add739209 100644 --- a/projects/caliper/engine/traverse.py +++ b/projects/caliper/engine/traverse.py @@ -14,8 +14,15 @@ MATRIXBENCHMARKING_MARKER = "settings.yaml" -def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: - """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode.""" +def discover_test_bases( + base_dir: Path, *, label_filter: dict[str, Any] | None = None +) -> list[TestBaseNode]: + """Walk base_dir; each directory containing MARKER or MATRIXBENCHMARKING_MARKER becomes a TestBaseNode. + + Args: + base_dir: Base directory to search + label_filter: Optional dict of label key-value pairs that must match for inclusion + """ base_dir = base_dir.resolve() if not base_dir.is_dir(): raise FileNotFoundError(f"Base directory does not exist: {base_dir}") @@ -32,10 +39,22 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: continue path = Path(dirpath) - marker_path = path / marker_found - labels = _load_labels( - marker_path, is_matrixbenchmarking=(marker_found == MATRIXBENCHMARKING_MARKER) - ) + + # Use hierarchical label loading for both marker types + if marker_found == MARKER: + labels = _load_hierarchical_labels(path, base_dir) + else: + # Use hierarchical loading for MatrixBenchmarking settings.yaml too + labels = _load_hierarchical_labels_matrixbenchmarking(path, base_dir) + + # Skip directory if skip: true is set in labels + if labels.get("skip") is True: + continue + + # Apply label filtering if specified + if label_filter is not None and not _matches_label_filter(labels, label_filter): + continue + nodes.append( TestBaseNode( directory=path, @@ -47,6 +66,180 @@ def discover_test_bases(base_dir: Path) -> list[TestBaseNode]: return sorted(nodes, key=lambda n: str(n.directory)) +def _load_hierarchical_labels(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge __test_labels__.yaml files hierarchically from base_dir down to test_dir. + + Merges in order: + 1. base_dir/__test_labels__.*.yaml (all variants) + 2. parent_dir/__test_labels__.*.yaml (all variants) + 3. test_dir/__test_labels__.*.yaml (all variants) + 4. test_dir/__test_labels__.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main __test_labels__.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge __test_labels__.*.yaml files (excluding plain __test_labels__.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all __test_labels__.*.yaml files (but not __test_labels__.yaml itself) + pattern = str(dir_path / "__test_labels__.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=False) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main __test_labels__.yaml from the test directory (final priority) + main_labels_path = test_dir / MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=False) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _load_hierarchical_labels_matrixbenchmarking(test_dir: Path, base_dir: Path) -> dict[str, Any]: + """Load and merge settings.yaml files hierarchically from base_dir down to test_dir for MatrixBenchmarking. + + Merges in order: + 1. base_dir/settings.*.yaml (all variants) + 2. parent_dir/settings.*.yaml (all variants) + 3. test_dir/settings.*.yaml (all variants) + 4. test_dir/settings.yaml (final, cannot be overridden) + + Later files override earlier ones, with the main settings.yaml having final priority. + """ + import glob + + merged_labels: dict[str, Any] = {} + + # Get all directories from base_dir down to test_dir (inclusive) + test_dir_abs = test_dir.resolve() + base_dir_abs = base_dir.resolve() + + # Build path from base_dir to test_dir + try: + rel_path = test_dir_abs.relative_to(base_dir_abs) + path_parts = [base_dir_abs] + [ + base_dir_abs / Path(*rel_path.parts[: i + 1]) for i in range(len(rel_path.parts)) + ] + except ValueError: + # test_dir is not under base_dir, just use test_dir + path_parts = [test_dir_abs] + + # For each directory in the hierarchy, merge settings.*.yaml files (excluding plain settings.yaml) + for dir_path in path_parts: + if not dir_path.is_dir(): + continue + + # Find all settings.*.yaml files (but not settings.yaml itself) + pattern = str(dir_path / "settings.*.yaml") + variant_files = sorted(glob.glob(pattern)) + + for variant_file in variant_files: + variant_path = Path(variant_file) + if variant_path.is_file(): + try: + variant_labels = _load_labels(variant_path, is_matrixbenchmarking=True) + # Merge the labels (later values override earlier ones) + _deep_merge_dict(merged_labels, variant_labels) + except (OSError, yaml.YAMLError, ValueError): + # Skip files that can't be loaded + pass + + # Finally, load the main settings.yaml from the test directory (final priority) + main_labels_path = test_dir / MATRIXBENCHMARKING_MARKER + if main_labels_path.is_file(): + try: + main_labels = _load_labels(main_labels_path, is_matrixbenchmarking=True) + _deep_merge_dict(merged_labels, main_labels) + except (OSError, yaml.YAMLError, ValueError): + # If main file can't be loaded, return what we have + pass + + return merged_labels + + +def _matches_label_filter(labels: dict[str, Any], label_filter: dict[str, Any]) -> bool: + """Check if labels match the specified filter criteria. + + Args: + labels: The hierarchically merged labels from the test directory + label_filter: Dict of key-value pairs that must match + + Returns: + True if all filter criteria match, False otherwise + + Examples: + _matches_label_filter({"labels": {"platform": "CKS", "version": "3.5"}}, + {"platform": "CKS"}) -> True + _matches_label_filter({"labels": {"platform": "OCP"}}, + {"platform": "CKS"}) -> False + """ + # Extract the labels section if it exists (for __test_labels__.yaml structure) + actual_labels = labels.get("labels", {}) if isinstance(labels.get("labels"), dict) else labels + + for filter_key, filter_value in label_filter.items(): + # Support nested key access with dot notation (e.g., "labels.platform") + if "." in filter_key: + keys = filter_key.split(".") + current = labels + for key in keys: + if not isinstance(current, dict) or key not in current: + return False + current = current[key] + if current != filter_value: + return False + else: + # Direct key access in labels section or root level + if filter_key not in actual_labels and filter_key not in labels: + return False + value = actual_labels.get(filter_key, labels.get(filter_key)) + if value != filter_value: + return False + + return True + + +def _deep_merge_dict(target: dict[str, Any], source: dict[str, Any]) -> None: + """Deep merge source dict into target dict, with source values taking precedence.""" + for key, value in source.items(): + if key in target and isinstance(target[key], dict) and isinstance(value, dict): + _deep_merge_dict(target[key], value) + else: + target[key] = value + + def _load_labels(path: Path, is_matrixbenchmarking: bool = False) -> dict[str, Any]: """Load labels from either __test_labels__.yaml or MatrixBenchmarking settings.yaml.""" raw = path.read_text(encoding="utf-8") diff --git a/projects/caliper/engine/visualize.py b/projects/caliper/engine/visualize.py index f85f6083c..529a3d03d 100644 --- a/projects/caliper/engine/visualize.py +++ b/projects/caliper/engine/visualize.py @@ -70,31 +70,61 @@ def run_visualize( use_cache: bool, cache_path: Path | None, ) -> list[str]: + # Convert include pairs to directory-level filter for discovery + directory_filter = None + if include_pairs: + inc = parse_filter_kv(include_pairs) + directory_filter = inc # Use include filters for directory discovery + + # Apply directory-level filtering during parsing model = run_parse( base_dir=base_dir, plugin_module=plugin_module, plugin=plugin, use_cache=use_cache, + label_filter=directory_filter, ) + + # Still apply record-level filtering for exclude filters and any remaining include filters inc = parse_filter_kv(include_pairs) exc = parse_filter_kv(exclude_pairs) - model.unified_result_records = filter_records( - model.unified_result_records, - include=inc, - exclude=exc, - ) + if exc: # Only filter records if there are exclude filters + model.unified_result_records = filter_records( + model.unified_result_records, + include={}, # Include filtering already done at directory level + exclude=exc, + ) cfg = resolve_visualize_config(base_dir, visualize_config_path) ids = resolve_report_ids( reports_csv=reports_csv, report_group=report_group, config=cfg, ) + + # Check if any report IDs were provided + if not ids: + raise ValueError( + "No report IDs provided. Use --reports with comma-separated report IDs " + "or --report-group with a valid visualize config file." + ) + output_dir.mkdir(parents=True, exist_ok=True) viz = plugin.visualize - return viz( + paths = viz( model, output_dir, ids, report_group, cfg, ) + + # Check for error conditions + if ids and not paths: + # Reports were requested but no files were generated + # This could be due to invalid plot types or other issues + raise ValueError( + f"No visualization files were generated despite requesting report IDs: {ids}. " + f"This may indicate unsupported plot types or other visualization errors." + ) + + return paths diff --git a/projects/caliper/orchestration/caliper_invocation.py b/projects/caliper/orchestration/caliper_invocation.py new file mode 100644 index 000000000..8acad1b07 --- /dev/null +++ b/projects/caliper/orchestration/caliper_invocation.py @@ -0,0 +1,741 @@ +""" +Caliper CLI command invocation functions. + +This module contains functions that build and execute caliper CLI commands +using subprocess execution with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +import tempfile +import time +import traceback +from pathlib import Path +from typing import Any + +from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_analyse_kpis_command, + build_kpi_csv_export_command, + build_kpi_generate_command, + build_parse_command, + build_s3_export_command, + build_s3_import_command, + build_visualize_command, +) +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) +from projects.core.library import env + +logger = logging.getLogger(__name__) + + +def _make_path_relative_to_base(file_path: str | Path, base_dir: Path) -> str: + """Convert absolute path to relative path from base directory. + + Args: + file_path: Absolute or relative file path + base_dir: Base directory to make path relative to + + Returns: + Relative path as string + """ + try: + path_obj = Path(file_path) + if path_obj.is_absolute() and base_dir: + base_obj = Path(base_dir) + if base_obj.is_absolute(): + try: + relative = path_obj.relative_to(base_obj) + return str(relative) + except ValueError: + # Path is not relative to base_dir, return as-is + return str(file_path) + return str(file_path) + except Exception: + return str(file_path) + + +def _execute_caliper_command( + command: list[str], + step_name: str, + status_file: Path, + step_logs_dir: Path, +) -> tuple[subprocess.CompletedProcess, dict[str, Any], Path]: + """Execute a Caliper CLI command and return result and status data. + + Args: + command: CLI command arguments list + step_name: Name for logging banners (e.g., "caliper parse", "caliper visualize") + status_file: Path to read status YAML from + step_logs_dir: Directory to create log file in + + Returns: + Tuple of (subprocess result, parsed status data, log file path) + """ + from projects.caliper.orchestration.cli_builder import log_caliper_start_banner + from projects.caliper.orchestration.postprocess_logging import ( + _handle_caliper_output_and_completion_with_header, + _write_step_footer_to_log_file, + _write_step_header_to_log_file, + ) + from projects.caliper.orchestration.step_logging import _get_next_step_index + + # Create log file path with proper step index + step_index = _get_next_step_index(step_logs_dir) + step_name_safe = step_name.replace(" ", "_") + log_file = step_logs_dir / f"{step_index:03d}__{step_name_safe}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + + # Create script file for debugging in step_scripts directory with index prefix + step_scripts_dir = env.ARTIFACT_DIR / "step_scripts" + step_scripts_dir.mkdir(parents=True, exist_ok=True) + script_file = step_scripts_dir / f"{step_index:03d}__{step_name_safe}.sh" + + # Log start banner and save script + log_caliper_start_banner(command, script_file, step_name.upper()) + + # Write header to log file + _write_step_header_to_log_file(log_file, command, step_name.upper()) + + # Execute command with combined stdout/stderr + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Combine stderr into stdout + text=True, + ) + + # Handle output, status parsing, and log completion banner + status_data = _handle_caliper_output_and_completion_with_header( + result, log_file, status_file, step_name.upper() + ) + + # Write footer to log file + _write_step_footer_to_log_file(log_file, result, step_name.upper()) + + return result, status_data, log_file + + +def run_artifacts_to_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Generate KPI JSON using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.enabled: + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } + if not postprocess_config.kpi.artifacts_to_kpis.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_kpis disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi generate", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_artifacts_to_ai_data( + postprocess_config, + plugin, + model, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export AI evaluation data using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.artifacts_to_ai_data.enabled: + return { + "status": "disabled", + "reason": "kpi.artifacts_to_ai_data disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "ai_data_dir": relative_path, # Expected by postprocess.py + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"AI evaluation export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_kpis_to_csv( + postprocess_config: CaliperOrchestrationPostprocessConfig, + plugin, + model, + output_dir: Path, + kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export KPI JSON to CSV using fork/exec subprocess execution.""" + + if not postprocess_config.kpi.csv.enabled: + return { + "status": "disabled", + "reason": "kpi.csv disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.kpi.csv.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" + + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + kpi_json_path=kpi_json_path, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + # Log the full traceback to help with debugging + full_traceback = traceback.format_exc() + logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_analyse_kpis( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, +) -> dict[str, Any]: + """Analyze KPIs using fork/exec subprocess execution.""" + + if not postprocess_config.analyze.enabled: + return { + "status": "disabled", + "reason": "analyze disabled", + "completed_at": time.time(), + "log_file": None, + } + + try: + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output + output_file.parent.mkdir(parents=True, exist_ok=True) + + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" + + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if result.returncode == 0 and status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("KPI analysis failed in run_analyse_kpis") + return {"status": "failed", "error": str(e), "completed_at": time.time(), "log_file": None} + + +def run_parse_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the parse step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.parse.enabled: + return True, {"status": "disabled", "reason": "parse disabled", "completed_at": time.time()} + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Parse completed successfully", + "test_directories": status_data.get("test_directories", []), + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper parse failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Parse step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during parse: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_visualize_step( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + step_logs_dir: Path, + output_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the visualize step if enabled. + + Returns: + Tuple of (success, step_result) + """ + if not config.visualize.enabled: + return True, { + "status": "disabled", + "reason": "visualize disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_visualize_command( + config=config, + tree_root=tree_root, + manifest_path=manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "Visualize completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("Caliper visualize failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("Visualize step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during visualize: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_import_step( + config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + step_logs_dir: Path, +) -> tuple[bool, dict[str, Any]]: + """Execute the S3 import step. + + Returns: + Tuple of (success, step_result) + """ + if not config.s3.import_.enabled: + return True, { + "status": "disabled", + "reason": "s3_import disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_import_command( + config=config, + status_file=status_file, + output_dir=output_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-import", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return True, { + "status": "success", + "detail": "S3 import completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 import failed: %s", error_msg) + return False, { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 import step execution failed") + return False, { + "status": "failed", + "detail": f"Exception during s3-import: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + +def run_s3_export_step( + config: CaliperOrchestrationPostprocessConfig, + step_logs_dir: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> dict[str, Any]: + """Execute the S3 export step. + + Returns: + Step result dictionary + """ + if not config.s3.export.enabled: + return { + "status": "disabled", + "reason": "s3_export disabled", + "completed_at": time.time(), + } + + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_s3_export_command( + config=config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + return { + "status": "success", + "detail": "S3 export completed successfully", + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + else: + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + logger.error("S3 export failed: %s", error_msg) + return { + "status": "failed", + "detail": error_msg, + "exit_code": result.returncode, + "completed_at": time.time(), + "log_file": log_file, + } + + except Exception as e: + logger.exception("S3 export step execution failed") + return { + "status": "failed", + "detail": f"Exception during s3-export: {str(e)}", + "completed_at": time.time(), + } + finally: + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass diff --git a/projects/caliper/orchestration/cli_builder.py b/projects/caliper/orchestration/cli_builder.py new file mode 100644 index 000000000..51cc03498 --- /dev/null +++ b/projects/caliper/orchestration/cli_builder.py @@ -0,0 +1,623 @@ +"""CLI command builder for Caliper orchestration fork/exec. + +Converts orchestration config objects to CLI argument lists for subprocess execution. +""" + +from __future__ import annotations + +import logging +import subprocess +from pathlib import Path + +from projects.caliper.orchestration.postprocess_config import ( + CaliperOrchestrationPostprocessConfig, +) + +logger = logging.getLogger(__name__) + +# Base command for all Caliper CLI invocations +_CALIPER_BASE_CMD = ["caliper"] + + +def build_parse_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper parse. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["parse"] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Parse-specific options + if not use_cache: + cmd.append("--no-cache") + + # Enable detailed parameter matrix output + cmd.append("--show-matrix") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_kpi_generate_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi generate. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output file for KPI data + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["kpi", "generate"] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Generate-specific options + cmd.extend(["--output", str(output_file)]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_visualize_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_dir: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper visualize. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_dir: Output directory for visualization files + use_cache: Whether to use parsing cache + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["visualize"] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # Visualize-specific options + cmd.extend(["--output-dir", str(output_dir)]) + + if config.visualize.reports: + cmd.extend(["--reports", config.visualize.reports]) + + if config.visualize.report_group: + cmd.extend(["--report-group", config.visualize.report_group]) + + if config.visualize.visualize_config: + # Resolve the visualize config path properly (same logic as _resolve_visualize_config_path) + viz_config_path = config.visualize.visualize_config + if viz_config_path: + from projects.core.library import env + + viz_path = Path(viz_config_path).expanduser() + if not viz_path.is_absolute(): + # If relative, resolve relative to FORGE_HOME like the original does + viz_path = env.FORGE_HOME / viz_path + cmd.extend(["--visualize-config", str(viz_path.resolve())]) + + # Include/exclude labels + for label in config.visualize.include_labels: + cmd.extend(["--include-label", label]) + + for label in config.visualize.exclude_labels: + cmd.extend(["--exclude-label", label]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_kpi_csv_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + input_file: Path, + output_file: Path, +) -> list[str]: + """Build CLI command for caliper kpi csv-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + input_file: Input KPI JSON file + output_file: Output CSV file + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["kpi", "csv-export"] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # CSV export specific options + cmd.extend(["--input", str(input_file)]) + cmd.extend(["--output", str(output_file)]) + + if config.kpi.kpis_to_csv.include_header_comments: + cmd.append("--include-header-comments") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_ai_eval_export_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + use_cache: bool = True, +) -> list[str]: + """Build CLI command for caliper ai-eval-export. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts + manifest_path: Optional manifest file path + status_file: Where to write status YAML + output_file: Output directory for AI eval data + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["ai-eval-export"] + + # Workspace options + cmd.extend(["--artifacts-dir", str(tree_root)]) + + if manifest_path: + cmd.extend(["--postprocess-config", str(manifest_path)]) + + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + + # AI eval export specific options + cmd.extend(["--output", str(output_file)]) + + if not use_cache: + cmd.append("--no-cache") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_import_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + output_dir: Path, +) -> list[str]: + """Build CLI command for caliper kpi s3-import. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + output_dir: Local output directory for downloads + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-import"] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory + from projects.caliper.cli.s3_export import build_s3_prefix + + import_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if import_prefix: + cmd.extend(["--prefix", import_prefix]) + + # Create subdirectory for historical data (matches original behavior) + import_dir = output_dir / s3_config.import_.output_dir + cmd.extend(["--output-dir", str(import_dir)]) + + # Import options + if s3_config.import_.include_kpis_json: + cmd.append("--include-kpis-json") + if s3_config.import_.include_kpis_csv: + cmd.append("--include-kpis-csv") + if s3_config.import_.include_ai_data: + cmd.append("--include-ai-data") + + if s3_config.import_.max_downloads: + cmd.extend(["--max-downloads", str(s3_config.import_.max_downloads)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_analyse_kpis_command( + config: CaliperOrchestrationPostprocessConfig, + tree_root: Path, + manifest_path: Path | None, + status_file: Path, + output_file: Path, + current_kpis_file: Path, + historical_kpis_dir: Path, +) -> list[str]: + """Build CLI command for caliper kpi analyse-kpis. + + Args: + config: Orchestration configuration + tree_root: Base directory for artifacts (unused, kept for compatibility) + manifest_path: Optional manifest file path (unused, kept for compatibility) + status_file: Where to write status YAML + output_file: Output file for analysis results + current_kpis_file: Current KPIs JSON file + historical_kpis_dir: Directory containing historical KPI files + + Returns: + List of command arguments for subprocess + """ + cmd = _CALIPER_BASE_CMD + ["kpi", "analyse-kpis"] + + # Analyse-kpis specific options + cmd.extend(["--output", str(output_file)]) + cmd.extend(["--current-kpis-file", str(current_kpis_file)]) + cmd.extend(["--historical-kpis-dir", str(historical_kpis_dir)]) + + # Plugin module is required + if config.plugin_module: + cmd.extend(["--plugin", config.plugin_module]) + else: + raise ValueError("Plugin module is required for KPI analysis") + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def build_s3_export_command( + config: CaliperOrchestrationPostprocessConfig, + status_file: Path, + kpis_file: Path | None = None, + csv_file: Path | None = None, + ai_data_dir: Path | None = None, + analysis_file: Path | None = None, +) -> list[str]: + """Build CLI command for caliper kpi s3-export. + + Args: + config: Orchestration configuration + status_file: Where to write status YAML + kpis_file: Optional KPIs JSON file + csv_file: Optional CSV file + ai_data_dir: Optional AI data directory + analysis_file: Optional analysis file + + Returns: + List of command arguments for subprocess + """ + s3_config = config.s3 + + cmd = _CALIPER_BASE_CMD + ["kpi", "s3-export"] + + # S3 configuration + cmd.extend(["--bucket", s3_config.bucket]) + + # Build prefix from instance and directory if available + from projects.caliper.cli.s3_export import build_s3_prefix + + export_prefix = build_s3_prefix( + instance=s3_config.instance, + directory=s3_config.directory, + ) + if export_prefix: + cmd.extend(["--prefix", export_prefix]) + + if s3_config.instance: + cmd.extend(["--instance", s3_config.instance]) + + if s3_config.directory: + cmd.extend(["--directory", s3_config.directory]) + + # File options + if kpis_file: + cmd.extend(["--kpis-file", str(kpis_file)]) + if csv_file: + cmd.extend(["--csv-file", str(csv_file)]) + if ai_data_dir: + cmd.extend(["--ai-data-dir", str(ai_data_dir)]) + if analysis_file: + cmd.extend(["--analysis-file", str(analysis_file)]) + + # Vault configuration + cmd.extend(["--vault", s3_config.vault.name]) + cmd.extend(["--aws-credentials-file", s3_config.vault.aws_credentials_file]) + + # Status file for orchestration + cmd.extend(["--status-file", str(status_file)]) + + return cmd + + +def save_command_script(command: list[str], script_path: Path) -> None: + """Save executed command to a shell script for debugging. + + Args: + command: Command arguments list + script_path: Where to save the script + """ + script_path.parent.mkdir(parents=True, exist_ok=True) + + # Convert command to a readable shell script + script_content = "#!/bin/bash\n" + script_content += "# Generated command for debugging\n" + script_content += "# Run this command to reproduce the step manually\n\n" + + # Format command with argument pairs on separate lines + formatted_lines = _format_command_for_script(command) + script_content += formatted_lines + "\n" + + with open(script_path, "w", encoding="utf-8") as f: + f.write(script_content) + + # Make executable + script_path.chmod(0o755) + + +def _format_command_for_script(command: list[str]) -> str: + """Format command for script file with argument pairs on separate lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for shell script + """ + if not command: + return "" + + def escape_arg(arg: str) -> str: + """Escape shell arguments that need quoting.""" + if " " in arg or any(c in arg for c in ["$", "`", '"', "'"]): + return f'"{arg}"' + return arg + + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(escape_arg(command[i])) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) with line continuations + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f" {escape_arg(command[i])} {escape_arg(command[i + 1])}") + i += 2 + else: + # Boolean flag + lines.append(f" {escape_arg(command[i])}") + i += 1 + else: + # Standalone argument + lines.append(f" {escape_arg(command[i])}") + i += 1 + + # Join with line continuations + if len(lines) == 1: + return lines[0] + else: + return lines[0] + " \\\n" + " \\\n".join(lines[1:]) + + +def log_caliper_start_banner(command: list[str], script_path: Path, step_name: str) -> None: + """Log start banner and save command script for Caliper execution. + + Args: + command: Command arguments list + script_path: Where to save the executable script + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + """ + # Save command script for debugging + save_command_script(command, script_path) + + # Format command with argument pairs on new lines + formatted_command = _format_command_for_display(command) + + # Log banner indicating Caliper is starting + logger.info("=" * 60) + logger.info("🚀 STARTING CALIPER %s (fork/exec)", step_name.upper()) + logger.info(f"📄 Command:\n{formatted_command}") + logger.info("=" * 60) + + +def _format_command_for_display(command: list[str]) -> str: + """Format command for readable display with argument pairs on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string + """ + if not command: + return "" + + # Start with base command (python -m module subcommand) + lines = [] + i = 0 + + # Handle base command parts until we hit options + base_parts = [] + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle option pairs (--flag value) + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return f" \\\n{len('caliper ') * ' '}".join(lines).strip() + + +def handle_caliper_output_and_completion( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Write output to log file + with open(log_file, "w", encoding="utf-8") as log_f: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + status_yaml = yaml.dump(status_data, default_flow_style=False, sort_keys=False) + for line in status_yaml.strip().splitlines(): + logger.info(" %s", line) + + logger.info("=" * 60) + + return status_data diff --git a/projects/caliper/orchestration/mlflow_verifier.py b/projects/caliper/orchestration/mlflow_verifier.py new file mode 100644 index 000000000..bfcdb56e0 --- /dev/null +++ b/projects/caliper/orchestration/mlflow_verifier.py @@ -0,0 +1,151 @@ +"""Reusable MLflow nightly verifier. + +Connects to the MLflow tracking server, queries runs in the project's +experiment, and extracts the version from the most recent run name. + +Projects reuse this by subclassing in their nightly_resolvers/ package +and setting NAME = "mlflow". The version extraction pattern is read from +``nightly.mlflow_verifier.version_pattern`` (regex with one capture group) +and an optional ``nightly.mlflow_verifier.strip_prefix``. + +All MLflow connection settings come from the caliper export config that +every FORGE project already has. +""" + +from __future__ import annotations + +import logging +import os +import re +import time +import warnings + +from projects.caliper.engine.file_export.mlflow_secrets import ( + load_mlflow_secrets_yaml, + mlflow_connection_env, + validate_mlflow_secrets, +) +from projects.core.library import vault +from projects.core.library.config import requires +from projects.core.nightly.base_verifier import NightlyVerifier + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 15 + + +class CaliperMLflowVerifier(NightlyVerifier): + """Check MLflow experiment runs for the last tested version. + + Config requirements (via @requires on __init__): + nightly.mlflow_verifier.experiment (experiment to query for previous runs) + caliper.export.backend.mlflow.config.workspace + caliper.export.backend.mlflow.secrets.vault (dict with 'name' and 'mlflow_secret') + nightly.mlflow_verifier.version_pattern (regex with one capture group) + nightly.mlflow_verifier.strip_prefix (prefix to remove from captured version) + """ + + @requires( + experiment="nightly.mlflow_verifier.experiment", + workspace="caliper.export.backend.mlflow.config.workspace", + vault_cfg="caliper.export.backend.mlflow.secrets.vault", + version_pattern="nightly.mlflow_verifier.version_pattern", + strip_prefix="nightly.mlflow_verifier.strip_prefix", + ) + def __init__(self, _cfg): + self.experiment = _cfg.experiment + self.workspace = _cfg.workspace + self.secret_name = _cfg.vault_cfg["name"] + self.secret_file = _cfg.vault_cfg["mlflow_secret"] + self._version_re = re.compile(_cfg.version_pattern) + self._strip_prefix = _cfg.strip_prefix or "" + + def get_last_tested_version(self) -> str: + connection = self._load_connection() + + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + with mlflow_connection_env(connection): + return self._query_latest_run() + except Exception as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "MLflow attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"MLflow verifier failed after {MAX_ATTEMPTS} attempts") from last_error + + def _load_connection(self) -> dict: + secret_path = vault.get_vault_content_path(self.secret_name, self.secret_file) + + if secret_path is None or not secret_path.exists(): + raise FileNotFoundError( + f"MLflow secret '{self.secret_name}/{self.secret_file}' not found in vault. " + "Ensure the vault configuration is correct." + ) + + data = load_mlflow_secrets_yaml(secret_path) + validate_mlflow_secrets(data) + return data + + def _query_latest_run(self) -> str: + import mlflow + import urllib3 + + warnings.filterwarnings("ignore", category=urllib3.exceptions.InsecureRequestWarning) + + previous_workspace = os.environ.get("MLFLOW_WORKSPACE") + try: + if self.workspace: + os.environ["MLFLOW_WORKSPACE"] = self.workspace + + client = mlflow.tracking.MlflowClient() + + experiment = client.get_experiment_by_name(self.experiment) + if experiment is None: + raise ValueError( + f"MLflow experiment '{self.experiment}' not found. " + "Create the experiment first or check the configuration." + ) + + runs = client.search_runs( + experiment_ids=[experiment.experiment_id], + order_by=["start_time DESC"], + max_results=1, + ) + + if not runs: + raise ValueError(f"No runs found in MLflow experiment '{self.experiment}'.") + + latest_run = runs[0] + run_name = latest_run.info.run_name or "" + logger.info("Latest MLflow run: %s", run_name) + + return self._extract_version(run_name) + finally: + if previous_workspace is None: + os.environ.pop("MLFLOW_WORKSPACE", None) + else: + os.environ["MLFLOW_WORKSPACE"] = previous_workspace + + def _extract_version(self, run_name: str) -> str: + match = self._version_re.search(run_name) + if not match: + raise ValueError( + f"Could not extract version from run name '{run_name}' " + f"using pattern '{self._version_re.pattern}'." + ) + + version = match.group(1) + + if self._strip_prefix and version.startswith(self._strip_prefix): + return version[len(self._strip_prefix) :] + + return version diff --git a/projects/caliper/orchestration/notification.py b/projects/caliper/orchestration/notification.py index c1e8fe46f..9a8158842 100644 --- a/projects/caliper/orchestration/notification.py +++ b/projects/caliper/orchestration/notification.py @@ -27,6 +27,7 @@ class PostprocessStepResult: uploaded_files: int | None = None downloaded_files: int | None = None failed_files: int | None = None + log_file: str | None = None # relative path to step log file @dataclass @@ -76,31 +77,33 @@ def format_postprocess_status_notification( for step_name, step_result in sorted_steps: step_emoji = _get_step_emoji(step_result.status) - lines.append(f"- {step_emoji} **{step_name}**: `{step_result.status}`") - # Add step message if available (but not for warning/failed steps to avoid duplication) - if step_result.message and step_result.status not in ("warning", "failed", "failure"): - lines.append(f" > {step_result.message}") - - # Add reason for skipped steps - if step_result.status in ("skipped", "disabled") and step_result.reason: - lines.append(f" > {step_result.reason}") - - # Add error message for failed steps + # Create step name as link to log file if available + if step_result.log_file and get_file_link: + try: + log_url = get_file_link(step_result.log_file) + step_name_display = f"[**{step_name}**]({log_url})" + except Exception: + # Fallback to plain text if link generation fails + step_name_display = f"**{step_name}**" + else: + step_name_display = f"**{step_name}**" + + # Format step with message if available + if step_result.message: + lines.append( + f"- {step_emoji} {step_name_display}: `{step_result.status}` * {step_result.message}" + ) + else: + lines.append(f"- {step_emoji} {step_name_display}: `{step_result.status}`") + + # Add reason for skipped steps (only if no message already shown) if ( - step_result.status in ("failed", "failure") - and hasattr(step_result, "error") - and step_result.error + step_result.status in ("skipped", "disabled") + and step_result.reason + and not step_result.message ): - lines.append(f" > ❌ {step_result.error}") - - # Add warning message for warning steps - if ( - step_result.status == "warning" - and hasattr(step_result, "message") - and step_result.message - ): - lines.append(f" > ⚠️ {step_result.message}") + lines.append(f" > {step_result.reason}") # Add specific file links for certain steps if step_result.status == "success" and get_file_link: @@ -208,7 +211,9 @@ def format_postprocess_status_notification( step_result, "paths", None ) if file_paths and get_file_link: - lines.extend(_format_step_file_links(step_name, file_paths, get_file_link)) + lines.extend( + _format_step_file_links(step_name, file_paths, get_file_link, step_result) + ) return "\n".join(lines) if lines else "" @@ -253,7 +258,7 @@ def _get_step_emoji(status: str) -> str: def _format_step_file_links( - step_name: str, file_paths: list[str], get_file_link: callable + step_name: str, file_paths: list[str], get_file_link: callable, step_result=None ) -> list[str]: """Format file paths as clickable links using the provided callback. @@ -261,6 +266,7 @@ def _format_step_file_links( step_name: Name of the step file_paths: List of relative file paths get_file_link: Callback function to generate URLs from file paths + step_result: Step result object containing additional metadata (optional) Returns: List of formatted link strings @@ -270,6 +276,9 @@ def _format_step_file_links( lines = [] + # Check if we need to combine output_dir with file paths (e.g., for visualize step) + output_dir = getattr(step_result, "output_dir", None) if step_result else None + # Group files by type for better organization file_groups = _group_files_by_type(file_paths) @@ -277,7 +286,13 @@ def _format_step_file_links( for file_type, files in file_groups.items(): for file_path in files: try: - file_url = get_file_link(file_path) + # Combine output_dir with file_path if available + if output_dir: + full_path = f"{output_dir}/{file_path}" + else: + full_path = file_path + + file_url = get_file_link(full_path) file_name = _get_display_name(file_path) emoji = "📊" if file_type == "visualization" else "📄" lines.append(f" - {emoji} [{file_name}]({file_url})") @@ -377,6 +392,7 @@ def parse_postprocess_result(status_data: dict) -> PostprocessResult | None: uploaded_files=step_data.get("uploaded_files"), downloaded_files=step_data.get("downloaded_files"), failed_files=step_data.get("failed_files"), + log_file=step_data.get("log_file"), ) return PostprocessResult( diff --git a/projects/caliper/orchestration/postprocess.py b/projects/caliper/orchestration/postprocess.py index ee285878e..41f1306af 100644 --- a/projects/caliper/orchestration/postprocess.py +++ b/projects/caliper/orchestration/postprocess.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import logging import time import traceback @@ -17,14 +18,19 @@ from pydantic import ValidationError -from projects.caliper.cli.orchestration_entrypoints import ( - analyze_kpis_entrypoint, - get_kpi_functions_entrypoint, - load_plugin_entrypoint, - parse_entrypoint, - run_visualize_entrypoint, - s3_export_entrypoint, - s3_import_entrypoint, +from projects.caliper.orchestration.caliper_invocation import ( + _execute_caliper_command, + run_analyse_kpis, +) +from projects.caliper.orchestration.cli_builder import ( + build_ai_eval_export_command, + build_analyse_kpis_command, + build_kpi_csv_export_command, + build_kpi_generate_command, + build_parse_command, + build_s3_export_command, + build_s3_import_command, + build_visualize_command, ) from projects.caliper.orchestration.postprocess_config import ( CaliperOrchestrationPostprocessConfig, @@ -37,13 +43,6 @@ ) from projects.caliper.orchestration.step_logging import ( cleanup_step_logging, - log_ai_data_command, - log_artifacts_to_kpis_command, - log_kpis_to_csv_command, - log_parse_command, - log_s3_export_command, - log_visualize_command, - step_logging, ) from projects.core.library import env @@ -93,13 +92,16 @@ def _resolve_visualize_output_dir( raw: str | None, ) -> Path: if raw is None or not str(raw).strip(): - raise ValueError( - "caliper.postprocess.visualize.output_dir is required when no explicit visualize_output_directory is provided" - ) + # If empty, use env.ARTIFACT_DIR + return env.ARTIFACT_DIR + p = Path(raw).expanduser() if p.is_absolute(): + # If absolute, don't touch return p.resolve() - raise ValueError("caliper.postprocess.visualize.output_dir must be an absolute path") + else: + # If relative, use env.ARTIFACT_DIR / output_dir + return (env.ARTIFACT_DIR / p).resolve() def _resolve_visualize_config_path( @@ -117,264 +119,180 @@ def _resolve_visualize_config_path( return (env.FORGE_HOME / p).resolve() - -# _load_plugin function removed - now using load_plugin_entrypoint - - -def _transform_kpis_to_hierarchical_format(kpis: list[dict], model) -> dict: - """ - Transform flat KPI list into hierarchical JSON structure. - - Groups KPIs by test (run_id), extracting common labels and organizing - KPI metadata (name, help, unit, etc.) for improved readability. - - Args: - kpis: List of flat KPI records from compute_kpis - model: Unified model for accessing plugin metadata - - Returns: - Hierarchical JSON structure organized by test - """ - from collections import defaultdict - - if not kpis: - return {"schema_version": "2", "tests": []} - - # Group KPIs by test (run_id) - tests_data = defaultdict(lambda: {"kpis": [], "labels": {}, "metadata": {}}) - - # Get KPI function metadata from the plugin module - kpi_functions = get_kpi_functions_entrypoint(model.plugin_module) - - for kpi in kpis: - run_id = kpi.get("run_id", "unknown") - test_data = tests_data[run_id] - - # Extract common labels (excluding KPI-specific ones) - kpi_labels = kpi.get("labels", {}) - test_labels = { - k: v for k, v in kpi_labels.items() if k not in ["higher_is_better"] - } # Exclude KPI-specific labels - - # Merge test labels (they should be the same for all KPIs in a test) - if not test_data["labels"]: - test_data["labels"] = test_labels - - # Store test metadata from first KPI - if not test_data["metadata"]: - test_data["metadata"] = { - "timestamp": kpi.get("timestamp"), - "source": kpi.get("source", {}), - "run_id": run_id, - } - - # Create KPI record with metadata - kpi_id = kpi.get("kpi_id") - raw_value = kpi.get("value") - - # Transform 2D KPI values to structured format - if isinstance(raw_value, list) and raw_value and len(raw_value) > 0: - # Check if this looks like 2D data: list of tuples/lists with 2 elements - first_item = raw_value[0] - if isinstance(first_item, list | tuple) and len(first_item) == 2: - try: - # Convert list of tuples [(x1, y1), (x2, y2), ...] to structured format - structured_value = { - "data_points": [{"x": float(x), "y": float(y)} for x, y in raw_value], - "count": len(raw_value), - } - final_value = structured_value - except (ValueError, TypeError, IndexError): - # If conversion fails, keep original value - final_value = raw_value - else: - final_value = raw_value - else: - final_value = raw_value - - kpi_record = { - "id": kpi_id, - "value": final_value, - "unit": kpi.get("unit"), - "higher_is_better": kpi_labels.get("higher_is_better", True), - } - - # Add KPI metadata from function decorator if available - if kpi_id in kpi_functions: - func = kpi_functions[kpi_id] - kpi_record.update( - { - "name": ( - func.__doc__.replace(" KPI.", "") - if func.__doc__ - else kpi_id.replace("_", " ").title() - ), - "help": getattr(func, "_kpi_help", ""), - } - ) - - # Add 2D-specific metadata if present - if getattr(func, "_kpi_is_2d", False): - kpi_record.update( - { - "is_2d": True, - "x_unit": getattr(func, "_kpi_x_unit", ""), - "x_help": getattr(func, "_kpi_x_help", ""), - "y_unit": getattr(func, "_kpi_y_unit", None) or kpi_record["unit"], - "y_help": getattr(func, "_kpi_y_help", None) - or getattr(func, "_kpi_help", ""), - } - ) - else: - kpi_record["is_2d"] = False - - # Add formatting info if available - if hasattr(func, "_kpi_format"): - kpi_record["format"] = func._kpi_format - else: - # Fallback if no function metadata available - kpi_record.update( - { - "name": kpi_id.replace("_", " ").title(), - "help": f"KPI: {kpi_id}", - "is_2d": isinstance(kpi.get("value"), list), - } - ) - - test_data["kpis"].append(kpi_record) - - # Convert to final structure - tests_list = [] - for run_id, test_data in tests_data.items(): - tests_list.append( - { - "run_id": run_id, - "labels": test_data["labels"], - "metadata": test_data["metadata"], - "kpis": test_data["kpis"], - } - ) - - return {"schema_version": "2", "tests": tests_list} + # _transform_kpis_to_hierarchical_format function removed - CLI commands now handle format conversion def _run_artifacts_to_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Generate KPI JSON using the plugin's compute_kpis method.""" + """Generate KPI JSON using fork/exec subprocess execution.""" if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } if not postprocess_config.kpi.artifacts_to_kpis.enabled: return { "status": "disabled", "reason": "kpi.artifacts_to_kpis disabled", "completed_at": time.time(), + "log_file": None, } try: - # Write KPI JSON - output_file = output_dir / postprocess_config.kpi.artifacts_to_kpis.output + # Prepare paths — reject absolute or parent-traversal output names + configured_output = postprocess_config.kpi.artifacts_to_kpis.output + if Path(configured_output).is_absolute() or ".." in Path(configured_output).parts: + raise ValueError( + f"kpi.artifacts_to_kpis.output must be a relative path without '..': {configured_output}" + ) + output_file = output_dir / configured_output + output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_artifacts_to_kpis_command( - base_dir=base_dir, - plugin_module=plugin_module, + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_generate_status.yaml" + + # Build CLI command + command = build_kpi_generate_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, ) - kpis = plugin.compute_kpis(model) - output_file.parent.mkdir(parents=True, exist_ok=True) - - import json + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi generate", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) - # Transform flat KPI list into hierarchical structure - hierarchical_data = _transform_kpis_to_hierarchical_format(kpis, model) + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + relative_path = _make_path_relative_to_base(output_file, env.ARTIFACT_DIR) + logger.info( + f"KPI generate: output_file={output_file}, env.ARTIFACT_DIR={env.ARTIFACT_DIR}, relative_path={relative_path}" + ) + return { + "status": "success", + "output_file": relative_path, + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } - with open(output_file, "w") as f: - json.dump(hierarchical_data, f, indent=2) + except Exception as e: + # Log the full traceback to help with debugging + import traceback - logger.info( - f"Generated hierarchical KPI structure with {len(kpis)} total records in {output_file}" - ) + full_traceback = traceback.format_exc() + logger.error(f"KPI generation failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return { - "status": "success", - "kpi_count": len(kpis), - "output_file": _make_path_relative_to_base(output_file, output_dir), + "status": "failed", + "message": str(e), "completed_at": time.time(), + "log_file": None, } - except Exception as e: - logger.error(f"KPI generation failed: {e}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} -# _run_s3_import function removed - now using s3_import_entrypoint directly +# _run_s3_import function removed - now using fork/exec subprocess execution directly def _run_artifacts_to_ai_data( postprocess_config, - plugin, - model, output_dir: Path, plugin_module: str, base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export AI evaluation payload with structured directories and copied artifacts.""" + """Export AI evaluation payload using fork/exec subprocess execution.""" try: - # Always log the step header, even for skipped operations + # Create AI data directory and output file path ai_data_dir = output_dir / postprocess_config.kpi.artifacts_to_ai_data.output_dir + ai_data_dir.mkdir(parents=True, exist_ok=True) output_file = ai_data_dir / "ai_data_payload.json" - log_ai_data_command( - base_dir=base_dir, - plugin_module=plugin_module, + + # Create temporary status file for subprocess communication + status_file = output_dir / "ai_eval_export_status.yaml" + + # Build CLI command + command = build_ai_eval_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, output_file=output_file, + use_cache=True, # Default to using cache in orchestration ) - if not hasattr(plugin, "build_ai_data_payload"): + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper ai-eval-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): return { - "status": "skipped", - "reason": "plugin does not support AI evaluation", + "status": "success", + "output_file": status_data.get("output_file", str(ai_data_dir)), + "ai_data_dir": _make_path_relative_to_base(ai_data_dir, env.ARTIFACT_DIR), "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, } - # Create AI evaluation directory structure using configured output directory - ai_data_dir.mkdir(parents=True, exist_ok=True) - - # Build payload from plugin - payload = plugin.build_ai_data_payload(model) - - # Export structured test entries with artifact copying - exported_entries = _export_test_entries_with_artifacts(model, ai_data_dir, base_dir, plugin) - - # Add exported entries info to payload - payload["exported_test_entries"] = exported_entries - - # Write main AI eval payload - output_file.parent.mkdir(parents=True, exist_ok=True) - - import json - - with open(output_file, "w") as f: - json.dump(payload, f, indent=2) - - logger.info(f"Generated AI evaluation payload in {output_file}") - logger.info(f"Exported {len(exported_entries)} test entries with artifacts") + except Exception as e: + # Log the full traceback to help with debugging + import traceback + full_traceback = traceback.format_exc() + logger.error(f"AI eval export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") return { - "status": "success", - "output_file": _make_path_relative_to_base(output_file, output_dir), - "ai_data_dir": _make_path_relative_to_base(ai_data_dir, output_dir), - "exported_entries": len(exported_entries), + "status": "failed", + "message": str(e), "completed_at": time.time(), + "log_file": None, } - except Exception as e: - logger.error(f"AI eval export failed: {e}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} def _load_test_labels(test_dir: Path) -> dict[str, Any]: @@ -403,142 +321,183 @@ def _load_test_labels(test_dir: Path) -> dict[str, Any]: return {} -def _export_test_entries_with_artifacts( - model, ai_data_dir: Path, base_dir: Path, plugin -) -> list[dict]: - """ - Export test entries by creating directories and copying specific artifacts. - - Args: - model: Unified model containing test results - ai_data_dir: Directory where test entries should be exported - base_dir: Base directory of the test artifacts (test directory) - plugin: Plugin instance to get artifact file list +def _run_kpis_to_csv( + postprocess_config: CaliperOrchestrationPostprocessConfig, + output_dir: Path, + kpi_json_path: Path, + base_dir: Path, + manifest_path: Path | None, + step_logs_dir: Path, +) -> dict[str, Any]: + """Export KPI data to CSV format using fork/exec subprocess execution.""" - Returns: - List of exported test entry information - """ - import shutil - - exported_entries = [] - - for idx, record in enumerate(model.unified_result_records): - # Create directory for this test entry - test_entry_dir = ai_data_dir / f"test_entry_{idx:03d}" - test_entry_dir.mkdir(parents=True, exist_ok=True) - - # Load test labels from __test_labels__.yaml if available - test_dir = base_dir / record.test_base_path - test_labels = _load_test_labels(test_dir) - - # Record test entry metadata - entry_info = { - "entry_id": f"test_entry_{idx:03d}", - "test_base_path": str(record.test_base_path), - "distinguishing_labels": record.distinguishing_labels, - "test_labels": test_labels, - "copied_files": [], - "missing_files": [], + if not postprocess_config.kpi.enabled: + return { + "status": "disabled", + "reason": "kpi disabled", + "completed_at": time.time(), + "log_file": None, + } + if not postprocess_config.kpi.kpis_to_csv.enabled: + return { + "status": "disabled", + "reason": "kpi.kpis_to_csv disabled", + "completed_at": time.time(), + "log_file": None, } - # Get artifact files specific to this test directory only (plugin is scoped to test directory) - relevant_files = plugin.get_ai_data_artifact_files_for_test(test_dir) - - logger.debug( - f"Test entry {idx}: found {len(relevant_files)} artifact files in test directory {test_dir}" - ) + try: + csv_output = postprocess_config.kpi.kpis_to_csv.output + if Path(csv_output).is_absolute() or ".." in Path(csv_output).parts: + raise ValueError( + f"kpi.kpis_to_csv.output must be a relative path without '..': {csv_output}" + ) + output_file = output_dir / csv_output + output_file.parent.mkdir(parents=True, exist_ok=True) - # Copy relevant files for this test entry (preserving directory structure) - for target_file in relevant_files: - # source is test_dir + test_relative_path, target is test_entry_dir + test_relative_path - source_file = test_dir / target_file - target_path = test_entry_dir / target_file + # Create temporary status file for subprocess communication + status_file = output_dir / "kpi_csv_export_status.yaml" - if source_file.exists(): - target_path.parent.mkdir(parents=True, exist_ok=True) + # Build CLI command + command = build_kpi_csv_export_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + input_file=kpi_json_path, + output_file=output_file, + ) - try: - shutil.copy2(source_file, target_path) - entry_info["copied_files"].append( - { - "source": str(source_file), - "target": str(target_path), - "relative_path": target_file, - "size_bytes": source_file.stat().st_size, - } - ) - logger.debug(f"Copied {source_file} -> {target_path}") - except Exception as e: - logger.warning(f"Failed to copy {source_file}: {e}") - entry_info["missing_files"].append({"file": str(source_file), "error": str(e)}) - else: - entry_info["missing_files"].append( - {"file": str(source_file), "error": "File does not exist"} - ) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi csv-export", + status_file=status_file, + step_logs_dir=step_logs_dir, + ) - # Write entry metadata - entry_metadata_file = test_entry_dir / "entry_metadata.json" - import json + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass - with open(entry_metadata_file, "w") as f: - json.dump(entry_info, f, indent=2) + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "kpi_count": status_data.get("kpi_count", 0), + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + "log_file": log_file, + } - exported_entries.append(entry_info) + except Exception as e: + # Log the full traceback to help with debugging + import traceback - return exported_entries + full_traceback = traceback.format_exc() + logger.error(f"KPI CSV export failed: {e}") + logger.error(f"Full traceback:\n{full_traceback}") + return { + "status": "failed", + "message": str(e), + "completed_at": time.time(), + "log_file": None, + } -def _run_kpis_to_csv( +def _run_analyse_kpis( postprocess_config: CaliperOrchestrationPostprocessConfig, - plugin, - model, output_dir: Path, - kpi_json_path: Path, + plugin_module: str, + base_dir: Path, + manifest_path: Path | None, + current_kpis_file: Path, + step_logs_dir: Path, ) -> dict[str, Any]: - """Export KPI data to CSV format using the plugin's compute_kpis method.""" + """Analyze KPIs using fork/exec subprocess execution.""" - if not postprocess_config.kpi.enabled: - return {"status": "disabled", "reason": "kpi disabled", "completed_at": time.time()} - if not postprocess_config.kpi.kpis_to_csv.enabled: + if not postprocess_config.analyze.enabled: return { "status": "disabled", - "reason": "kpi.kpis_to_csv disabled", + "reason": "analyze disabled", "completed_at": time.time(), + "log_file": None, } try: - # Compute KPIs from the model - kpi_records = plugin.compute_kpis(model) - - # Create output file path - output_file = output_dir / postprocess_config.kpi.kpis_to_csv.output + # Prepare paths + output_file = output_dir / postprocess_config.analyze.output output_file.parent.mkdir(parents=True, exist_ok=True) - # Log command to reproduce this step - log_kpis_to_csv_command( - input_path=kpi_json_path, - output_path=output_file, - ) + # Calculate historical KPIs directory path + historical_kpis_dir = Path(postprocess_config.analyze.historical_kpis) + if not historical_kpis_dir.is_absolute(): + historical_kpis_dir = output_dir / historical_kpis_dir + + # Create temporary status file for subprocess communication + status_file = output_dir / "analyse_kpis_status.yaml" - # Import and use the CSV exporter - from projects.guidellm.postprocess.guidellm.csv_export import quick_export_kpis_to_csv + # Build CLI command + command = build_analyse_kpis_command( + config=postprocess_config, + tree_root=base_dir, + manifest_path=manifest_path, + status_file=status_file, + output_file=output_file, + current_kpis_file=current_kpis_file, + historical_kpis_dir=historical_kpis_dir, + ) - result_path = quick_export_kpis_to_csv( - records=kpi_records, - output_path=output_file, - include_header_comments=postprocess_config.kpi.kpis_to_csv.include_header_comments, + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper kpi analyse-kpis", + status_file=status_file, + step_logs_dir=step_logs_dir, ) - logger.info(f"Exported {len(kpi_records)} KPI records to CSV: {result_path}") + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + return { + "status": "success", + "output_file": _make_path_relative_to_base(output_file, env.ARTIFACT_DIR), + "findings_count": status_data.get("findings_count", 0), + "regressions_count": status_data.get("regressions_count", 0), + "improvements_count": status_data.get("improvements_count", 0), + "completed_at": time.time(), + "log_file": log_file, + } + else: + return { + "status": "failed", + "error": result.get("error", "Analysis failed"), + "completed_at": time.time(), + "log_file": None, + } + + except Exception as e: + logger.exception("KPI analysis failed in _run_analyse_kpis") return { - "status": "success", - "kpi_count": len(kpi_records), - "output_file": _make_path_relative_to_base(result_path, output_dir), + "status": "failed", + "message": str(e), "completed_at": time.time(), + "log_file": None, } - except Exception as e: - logger.error(f"KPI CSV export failed: {e}") - return {"status": "failed", "error": str(e), "completed_at": time.time()} class CaliperPostprocessOrchestrator: @@ -563,13 +522,14 @@ def __init__( # State tracking self.steps: list[dict[str, Any]] = [] - self.parse_failed = False - self.visualize_failed = False - self.artifacts_to_kpis_failed = False - self.ai_data_failed = False - self.s3_import_failed = False - self.analyze_failed = False - self.s3_export_failed = False + # Initialize failure flags to None (skipped), will be set to False (success) or True (failure) + self.parse_failed = None + self.visualize_failed = None + self.artifacts_to_kpis_failed = None + self.ai_data_failed = None + self.s3_import_failed = None + self.analyze_failed = None + self.s3_export_failed = None # Configuration try: @@ -647,13 +607,20 @@ def _execute_orchestration(self) -> dict[str, Any]: logger.info("Starting postprocessing steps") self._run_parse_step() logger.info(f"After parse step: parse_failed={self.parse_failed}") - self._run_visualize_step() - logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") - self._run_kpi_and_ai_data_steps() - logger.info( - f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" - ) - logger.info("All postprocessing steps completed") + + # Abort pipeline if parse failed - no point in running other steps without data + if self.parse_failed: + logger.error("Parse step failed - aborting remaining postprocessing steps") + logger.info("Pipeline aborted due to parse failure") + else: + self._run_visualize_step() + logger.info(f"After visualize step: visualize_failed={self.visualize_failed}") + + self._run_kpi_and_ai_data_steps() + logger.info( + f"After KPI/AI steps: artifacts_to_kpis_failed={self.artifacts_to_kpis_failed}, ai_data_failed={self.ai_data_failed}" + ) + logger.info("All postprocessing steps completed") # Compute final status and build result final_status = self._compute_final_status() @@ -677,12 +644,24 @@ def _setup_paths(self) -> None: self.step_logs_dir = Path(env.ARTIFACT_DIR) self.step_logs_dir.mkdir(parents=True, exist_ok=True) - def _add_step(self, step_name: str, step_data: dict[str, Any]) -> None: + def _add_step( + self, step_name: str, step_data: dict[str, Any], log_file: Path | None = None + ) -> None: """Add a step result to the steps list.""" + if log_file: + # Make log file path relative to artifact root where logs are actually stored + try: + artifact_root = Path(env.ARTIFACT_DIR) + relative_log_path = log_file.relative_to(artifact_root) + step_data["log_file"] = str(relative_log_path) + except ValueError: + # If log file is not under artifact root, use absolute path + step_data["log_file"] = str(log_file) + self.steps.append({step_name: step_data}) def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, Any]) -> bool: - """Check step result status and set appropriate failure flag if needed. + """Check step result status and set appropriate failure flag. Args: step_name: Name of the step @@ -692,28 +671,40 @@ def _check_step_result_and_set_failure(self, step_name: str, result: dict[str, A True if step failed or warned, False if successful """ status = result.get("status") - if status in ("failed", "warning"): - # Map step names to their failure flags - step_failure_map = { - "artifacts_to_kpis": "artifacts_to_kpis_failed", - "artifacts_to_ai_data": "ai_data_failed", - "s3_import": "s3_import_failed", - "analyse_kpis": "analyze_failed", - "s3_export": "s3_export_failed", - } - failure_attr = step_failure_map.get(step_name) - if failure_attr: - setattr(self, failure_attr, True) + # Map step names to their failure flags + step_failure_map = { + "artifacts_to_kpis": "artifacts_to_kpis_failed", + "artifacts_to_ai_data": "ai_data_failed", + "s3_import": "s3_import_failed", + "analyse_kpis": "analyze_failed", + "s3_export": "s3_export_failed", + } + + failure_attr = step_failure_map.get(step_name) + if not failure_attr: + return False - if status == "warning": - warning_msg = result.get("message", "unknown warning") - logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") - elif status == "failed": - error_msg = result.get("error", "unknown error") - logger.error(f"Step '{step_name}' failed: {error_msg}") + if status in ("failed", "warning"): + setattr(self, failure_attr, True) + + if status == "warning": + warning_msg = result.get("message", "unknown warning") + logger.warning(f"Step '{step_name}' completed with warning: {warning_msg}") + elif status == "failed": + error_msg = result.get("message", "unknown error") + traceback_msg = result.get("traceback") + logger.error(f"Step '{step_name}' failed: {error_msg}") + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) return True + elif status == "success": + # Set flag to False for successful steps + setattr(self, failure_attr, False) + return False + + # For other statuses (disabled, etc.), leave flag as None return False def _get_step(self, step_name: str) -> dict[str, Any]: @@ -760,7 +751,7 @@ def _build_result(self, final_status: str, test_block: dict[str, Any]) -> dict[s return { "final_status": final_status, "success": success_value, - "base_directory": str(self.visualize_output_dir) if self.visualize_output_dir else None, + "base_directory": str(Path(env.ARTIFACT_DIR)), "test_phase": test_block, "steps": self.steps, } @@ -770,240 +761,278 @@ def _run_parse_step(self) -> None: if not self.config.parse.enabled: return - with step_logging("caliper_parse", self.step_logs_dir): - try: - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, - ) + import tempfile - # Log command to reproduce this step - log_parse_command( - base_dir=self.tree_root, - plugin_module=mod_str, - use_cache=not self.config.parse.no_cache, - manifest_path=self.manifest_path, - ) + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) + + try: + # Build CLI command + command = build_parse_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + use_cache=not self.config.parse.no_cache, + ) + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper parse", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + self.parse_failed = False self._add_step( "parse", { "status": "success", - "plugin_module": mod_str, - "record_count": len(model.unified_result_records), - "parse_cache_ref": model.parse_cache_ref, + "plugin_module": status_data.get("plugin_module", "unknown"), + "record_count": status_data.get("parsed_records", 0), + "test_directories": status_data.get("test_directories", []), + "test_directories_count": status_data.get("test_directories_count", 0), + "parse_cache_ref": status_data.get("cache_ref"), "completed_at": time.time(), }, + log_file, ) - except Exception as e: # noqa: BLE001 + else: self.parse_failed = True - logger.exception("Caliper parse failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + traceback_msg = (status_data or {}).get("traceback") + logger.error("Caliper parse failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "parse", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, + log_file, ) + except Exception as e: # noqa: BLE001 + self.parse_failed = True + logger.exception("Parse step execution failed") + self._add_step( + "parse", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_visualize_step(self) -> None: """Execute the visualize step if enabled.""" if not self.config.visualize.enabled: return - with step_logging("caliper_visualize", self.step_logs_dir): - try: - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path - ) + import tempfile - viz_cfg_path = _resolve_visualize_config_path( - self.config.visualize.visualize_config, - artifact_tree=self.tree_root, - ) + # Create status file for CLI output + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as status_f: + status_file = Path(status_f.name) - if self.visualize_output_dir is not None: - output_dir = self.visualize_output_dir.expanduser().resolve() - else: - output_dir = _resolve_visualize_output_dir( - self.config.visualize.output_dir, - ) - - # Log command to reproduce this step - log_visualize_command( - base_dir=self.tree_root, - plugin_module=mod_str, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, + try: + # Resolve visualize output directory + if self.visualize_output_dir is not None: + output_dir = self.visualize_output_dir.expanduser().resolve() + logger.info(f"Using explicit visualize output directory: {output_dir}") + else: + output_dir = _resolve_visualize_output_dir( + self.config.visualize.output_dir, ) - - paths = run_visualize_entrypoint( - base_dir=self.tree_root, - plugin_module=mod_str, - plugin=plugin, - output_dir=output_dir, - reports_csv=self.config.visualize.reports, - report_group=self.config.visualize.report_group, - visualize_config_path=viz_cfg_path, - include_pairs=tuple(self.config.visualize.include_labels), - exclude_pairs=tuple(self.config.visualize.exclude_labels), - use_cache=not self.config.parse.no_cache, - cache_path=self.cache_path, + logger.info( + f"Resolved visualize output directory from config '{self.config.visualize.output_dir}': {output_dir}" ) - # Convert paths to relative paths from output_dir + # Build CLI command + command = build_visualize_command( + config=self.config, + tree_root=self.tree_root, + manifest_path=self.manifest_path, + status_file=status_file, + output_dir=output_dir, + use_cache=not self.config.parse.no_cache, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper visualize", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) + + if result.returncode == 0 and status_data and status_data.get("success", False): + # Get output files and paths from status + output_files = status_data.get("output_files", []) + + # Convert absolute paths to relative paths from output_dir if needed relative_paths = [] - for path in paths: + for path in output_files: try: path_obj = Path(path) - relative_path = path_obj.relative_to(output_dir) - relative_paths.append(str(relative_path)) + if path_obj.is_absolute(): + relative_path = path_obj.relative_to(output_dir) + relative_paths.append(str(relative_path)) + else: + relative_paths.append(str(path)) except ValueError: # If path is not under output_dir, keep as-is relative_paths.append(str(path)) + # Calculate relative output_dir path from base_directory + try: + relative_output_dir = str(output_dir.relative_to(env.ARTIFACT_DIR)) + except ValueError: + # If output_dir is not under ARTIFACT_DIR, use absolute path + relative_output_dir = str(output_dir) + + self.visualize_failed = False self._add_step( "visualize", { "status": "success", - "plugin_module": mod_str, + "plugin_module": status_data.get("plugin_module", "unknown"), "output_files": relative_paths, + "output_dir": relative_output_dir, + "generated_files": status_data.get("generated_files", len(relative_paths)), "completed_at": time.time(), }, + log_file, ) - - except Exception as e: # noqa: BLE001 + else: self.visualize_failed = True - logger.exception("Caliper visualize failed") + error_msg = (status_data or {}).get( + "error", f"Command failed with exit code {result.returncode}" + ) + traceback_msg = (status_data or {}).get("traceback") + logger.error("Caliper visualize failed: %s", error_msg) + if traceback_msg: + logger.error("Full traceback:\n%s", traceback_msg) self._add_step( "visualize", { "status": "failure", - "detail": str(e), - "traceback": traceback.format_exc(), + "detail": error_msg, + "exit_code": result.returncode, "completed_at": time.time(), }, + log_file, ) + except Exception as e: # noqa: BLE001 + self.visualize_failed = True + logger.exception("Visualize step execution failed") + self._add_step( + "visualize", + { + "status": "failure", + "detail": str(e), + "traceback": traceback.format_exc(), + "completed_at": time.time(), + }, + None, # No log file if we couldn't even start + ) + finally: + # Clean up temporary status file + if status_file.exists(): + status_file.unlink() + def _run_kpi_and_ai_data_steps(self) -> None: """Execute KPI generation, CSV export, KPI export, and AI evaluation steps.""" if not self.config.kpi.enabled: return + # Setup output directory and module string with focused error handling try: - # Determine output directory - if self.config.visualize.enabled and self.visualize_output_dir: - output_dir = Path(self.visualize_output_dir) - else: - output_dir = Path(self.artifacts_dir) / "postprocess_output" - output_dir.mkdir(parents=True, exist_ok=True) - - # Load plugin and model - model, mod_str = parse_entrypoint( - self.config, - self.tree_root, - self.manifest_path, - use_cache=not self.config.parse.no_cache, - ) - mod_str, plugin = load_plugin_entrypoint( - self.config, tree_root=self.tree_root, manifest_path=self.manifest_path - ) - - # KPI JSON generation - self._run_artifacts_to_kpis_step(plugin, model, output_dir, mod_str) - - # KPI CSV export - self._run_kpis_to_csv_step(plugin, model, output_dir) - - # AI evaluation export - self._run_artifacts_to_ai_data_step(plugin, model, output_dir, mod_str) - - # S3 import (historical data) - self._run_s3_import_step(output_dir) - - # Analyze KPIs (current vs historical) - moved before S3 export - self._run_analyse_kpis_step(output_dir, mod_str) - - # S3 export - self._run_s3_export_step(output_dir) + # Determine output directory for KPI/AI data steps - use base artifact directory + output_dir = env.ARTIFACT_DIR + logger.info(f"KPI steps using base artifact directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + # Resolve plugin module string (this is just string manipulation, not engine access) + mod_str = self.config.plugin_module or "unknown" except Exception as e: completion_time = time.time() - logger.error(f"Failed to run KPI/AI eval operations: {e}") - self._add_step( + logger.error(f"Failed to setup KPI/AI eval operations: {e}") + # Only mark all steps as failed if basic setup fails + for step_name in [ "artifacts_to_kpis", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "kpis_to_csv", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "artifacts_to_ai_data", - { - "status": "failed", - "error": str(e), - "completed_at": completion_time, - }, - ) - self._add_step( "s3_import", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "analyse_kpis", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) - self._add_step( "s3_export", - { - "status": "skipped", - "reason": "failed to load plugin", - "completed_at": completion_time, - }, - ) + ]: + self._add_step( + step_name, + { + "status": "failed", + "error": f"Setup failed: {e}", + "completed_at": completion_time, + }, + ) self.artifacts_to_kpis_failed = True self.ai_data_failed = True self.s3_import_failed = True self.analyze_failed = True self.s3_export_failed = True + return - def _run_artifacts_to_kpis_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + # Run each step independently - each has its own error handling + # KPI JSON generation + self._run_artifacts_to_kpis_step(output_dir, mod_str) + + # Generate per-run metrics.json + parameters.json from kpis.json + self._run_kpis_to_metrics_step(output_dir) + + # KPI CSV export + self._run_kpis_to_csv_step(output_dir) + + # AI evaluation export + self._run_artifacts_to_ai_data_step(output_dir, mod_str) + + # S3 import (historical data) + self._run_s3_import_step(output_dir) + + # Analyze KPIs (current vs historical) - moved before S3 export + self._run_analyse_kpis_step(output_dir, mod_str) + + # S3 export + self._run_s3_export_step(output_dir) + + def _run_artifacts_to_kpis_step(self, output_dir: Path, mod_str: str) -> None: """Execute the KPI generation step.""" if self.config.kpi.artifacts_to_kpis.enabled: - with step_logging("caliper_artifacts_to_kpis", self.step_logs_dir): - result = _run_artifacts_to_kpis( - self.config, plugin, model, output_dir, mod_str, self.tree_root - ) - self._add_step("artifacts_to_kpis", result) - self._check_step_result_and_set_failure("artifacts_to_kpis", result) + result = _run_artifacts_to_kpis( + self.config, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_kpis", result, log_file) + self._check_step_result_and_set_failure("artifacts_to_kpis", result) else: self._add_step( "artifacts_to_kpis", @@ -1014,7 +1043,40 @@ def _run_artifacts_to_kpis_step( }, ) - def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> None: + def _run_kpis_to_metrics_step(self, output_dir: Path) -> None: + """Generate per-run metrics.json + parameters.json from kpis.json. + + Runs automatically after kpis.json generation succeeds. Writes + files into each test run directory so the MLflow export backend + picks them up via ``_log_metrics_and_params_from_tree``. + """ + kpi_step = self._get_step("artifacts_to_kpis") + if not kpi_step or kpi_step.get("status") != "success": + return + + kpis_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + if not kpis_json_path.is_file(): + logger.warning("kpis.json not found at %s, skipping metrics generation", kpis_json_path) + return + + try: + from projects.caliper.engine.kpi.metrics_from_kpis import generate_metrics_from_kpis + + result = generate_metrics_from_kpis(kpis_json_path, self.tree_root) + status = result.get("status", "unknown") + if status == "success": + logger.info( + "kpis-to-metrics: wrote metrics.json for %d test(s)", + result.get("tests_processed", 0), + ) + elif status == "skipped": + logger.info("kpis-to-metrics: skipped — %s", result.get("reason", "")) + else: + logger.warning("kpis-to-metrics: %s", result.get("error", "unknown error")) + except Exception as e: + logger.warning("kpis-to-metrics conversion failed (non-fatal): %s", e) + + def _run_kpis_to_csv_step(self, output_dir: Path) -> None: """Execute the KPI CSV export step.""" if not self.config.kpi.kpis_to_csv.enabled: self._add_step( @@ -1027,18 +1089,22 @@ def _run_kpis_to_csv_step(self, plugin: Any, model: Any, output_dir: Path) -> No ) return - with step_logging("caliper_kpis_to_csv", self.step_logs_dir): - # Path to the JSON file for reference in command logging - kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output - result = _run_kpis_to_csv(self.config, plugin, model, output_dir, kpi_json_path) - self._add_step("kpis_to_csv", result) - if result.get("status") == "failed": - # CSV export failure doesn't affect overall status - it's supplementary - logger.warning("KPI CSV export failed but continuing execution") - - def _run_artifacts_to_ai_data_step( - self, plugin: Any, model: Any, output_dir: Path, mod_str: str - ) -> None: + kpi_json_path = output_dir / self.config.kpi.artifacts_to_kpis.output + result = _run_kpis_to_csv( + self.config, + output_dir, + kpi_json_path, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("kpis_to_csv", result, log_file) + if result.get("status") == "failed": + # CSV export failure doesn't affect overall status - it's supplementary + logger.warning("KPI CSV export failed but continuing execution") + + def _run_artifacts_to_ai_data_step(self, output_dir: Path, mod_str: str) -> None: """Execute the AI evaluation export step.""" if not self.config.kpi.artifacts_to_ai_data.enabled: self._add_step( @@ -1051,22 +1117,29 @@ def _run_artifacts_to_ai_data_step( ) return - with step_logging("caliper_artifacts_to_ai_data", self.step_logs_dir): - try: - result = _run_artifacts_to_ai_data( - self.config, plugin, model, output_dir, mod_str, self.tree_root - ) - self._add_step("artifacts_to_ai_data", result) - logger.info(f"AI eval export result: {result}") + try: + result = _run_artifacts_to_ai_data( + self.config, + output_dir, + mod_str, + self.tree_root, + self.manifest_path, + self.step_logs_dir, + ) + log_file = result.pop("log_file", None) + self._add_step("artifacts_to_ai_data", result, log_file) + + logger.info("AI eval export result:") + logger.info(json.dumps(result, indent=2, default=str)) - # Check if the result indicates failure or warning - self._check_step_result_and_set_failure("artifacts_to_ai_data", result) + # Check if the result indicates failure or warning + self._check_step_result_and_set_failure("artifacts_to_ai_data", result) - except Exception as e: - logger.exception("AI eval export failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("artifacts_to_ai_data", step_result) - self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) + except Exception as e: + logger.exception("AI eval export failed") + step_result = {"status": "failed", "error": str(e)} + self._add_step("artifacts_to_ai_data", step_result, None) + self._check_step_result_and_set_failure("artifacts_to_ai_data", step_result) def _run_s3_import_step(self, output_dir: Path) -> None: """Execute the S3 import step.""" @@ -1081,27 +1154,66 @@ def _run_s3_import_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_import", self.step_logs_dir): + try: + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_import_status.yaml" + + # Build CLI command + command = build_s3_import_command( + config=self.config, + status_file=status_file, + output_dir=output_dir, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 import", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) + + # Clean up temporary status file try: - result = s3_import_entrypoint(self.config, output_dir) + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + # Get the actual import directory (where files were downloaded) + import_dir = output_dir / self.config.s3.import_.output_dir + step_result = { + "status": "success", + "output_dir": _make_path_relative_to_base(import_dir, env.ARTIFACT_DIR), + "file_count": status_data.get("file_count", 0), + "completed_at": time.time(), + } + if status_data.get("warning"): + step_result["warning"] = status_data["warning"] + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] + else: + step_result = { + "status": "failed", + "error": status_data.get("error", "Unknown error"), + "completed_at": time.time(), + } + if status_data.get("imported_path"): + step_result["imported_path"] = status_data["imported_path"] - # Standardize field names and convert paths to relative - if "import_dir" in result: - # Rename import_dir to output_dir and make relative - result["output_dir"] = _make_path_relative_to_base( - result.pop("import_dir"), output_dir - ) + self._add_step("s3_import", step_result, log_file) - self._add_step("s3_import", result) - logger.info(f"S3 import result: {result}") + logger.info("S3 import result:") + logger.info(json.dumps(step_result, indent=2, default=str)) - self._check_step_result_and_set_failure("s3_import", result) + self._check_step_result_and_set_failure("s3_import", step_result) - except Exception as e: - logger.exception("S3 import failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("s3_import", step_result) - self._check_step_result_and_set_failure("s3_import", step_result) + except Exception as e: + logger.exception("S3 import failed") + step_result = {"status": "failed", "error": str(e), "completed_at": time.time()} + self._add_step("s3_import", step_result, None) + self._check_step_result_and_set_failure("s3_import", step_result) def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: """Execute the KPI analysis step.""" @@ -1116,58 +1228,43 @@ def _run_analyse_kpis_step(self, output_dir: Path, plugin_module: str) -> None: ) return - # Get KPI file path from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if not artifacts_to_kpis_step or artifacts_to_kpis_step.get("status") != "success": - self._add_step( - "analyse_kpis", - { - "status": "failed", - "error": "artifacts_to_kpis step did not complete successfully", - "completed_at": time.time(), - }, - ) - self.analyze_failed = True - return + # Get current KPI file path from config + current_kpis_file = Path(self.config.analyze.current_kpis) + if not current_kpis_file.is_absolute(): + current_kpis_path = output_dir / current_kpis_file + else: + current_kpis_path = current_kpis_file - current_kpis_file = artifacts_to_kpis_step.get("output_file") - if not current_kpis_file: + # Check if current KPI file exists + if not current_kpis_path.exists(): self._add_step( "analyse_kpis", { "status": "failed", - "error": "No output file found in artifacts_to_kpis step", + "error": f"Current KPI file not found: {current_kpis_path}", "completed_at": time.time(), }, ) self.analyze_failed = True return - with step_logging("caliper_analyse_kpis", self.step_logs_dir): - try: - # Convert relative path to absolute for analyze function - current_kpis_path = output_dir / current_kpis_file - - result = analyze_kpis_entrypoint( - self.config, plugin_module, self.tree_root, output_dir, current_kpis_path - ) - - # Make output_file path relative - if "output_file" in result: - result["output_file"] = _make_path_relative_to_base( - result["output_file"], output_dir - ) + result = run_analyse_kpis( + postprocess_config=self.config, + output_dir=output_dir, + plugin_module=plugin_module, + base_dir=self.tree_root, + manifest_path=self.manifest_path, + current_kpis_file=current_kpis_path, + step_logs_dir=self.step_logs_dir, + ) - self._add_step("analyse_kpis", result) - logger.info(f"KPI analysis result: {result}") + log_file = result.pop("log_file", None) + self._add_step("analyse_kpis", result, log_file) - self._check_step_result_and_set_failure("analyse_kpis", result) + logger.info("KPI analysis result:") + logger.info(json.dumps(result, indent=2, default=str)) - except Exception as e: - logger.exception("KPI analysis failed") - step_result = {"status": "failed", "error": str(e)} - self._add_step("analyse_kpis", step_result) - self._check_step_result_and_set_failure("analyse_kpis", step_result) + self._check_step_result_and_set_failure("analyse_kpis", result) def _run_s3_export_step(self, output_dir: Path) -> None: """Execute the S3 export step.""" @@ -1182,119 +1279,136 @@ def _run_s3_export_step(self, output_dir: Path) -> None: ) return - with step_logging("caliper_s3_export", self.step_logs_dir): - try: - # Collect file paths from previous steps - kpis_file = None - csv_file = None - ai_data_dir = None - analysis_file = None - - # Get KPI JSON file from artifacts_to_kpis step - artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") - if ( - artifacts_to_kpis_step - and artifacts_to_kpis_step.get("status") == "success" - and artifacts_to_kpis_step.get("output_file") - ): - kpis_file = output_dir / artifacts_to_kpis_step["output_file"] - - # Get CSV file from kpis_to_csv step - kpis_to_csv_step = self._get_step("kpis_to_csv") - if ( - kpis_to_csv_step - and kpis_to_csv_step.get("status") == "success" - and kpis_to_csv_step.get("output_file") - ): - csv_file = output_dir / kpis_to_csv_step["output_file"] - - # Get AI data directory from artifacts_to_ai_data step - ai_data_step = self._get_step("artifacts_to_ai_data") - logger.info(f"AI data step found: {ai_data_step}") - if ( - ai_data_step - and ai_data_step.get("status") == "success" - and ai_data_step.get("ai_data_dir") - ): - ai_data_dir = output_dir / ai_data_step["ai_data_dir"] - logger.info(f"AI data directory set to: {ai_data_dir}") - else: - logger.warning( - f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" - ) - - # Get analysis file from analyse_kpis step - analyze_step = self._get_step("analyse_kpis") - if ( - analyze_step - and analyze_step.get("status") in ("success", "warning") - and analyze_step.get("output_file") - ): - analysis_file = output_dir / analyze_step["output_file"] - - # Log the CLI command to reproduce this step - s3_parent_config = self.config.s3 - export_config = s3_parent_config.export - export_path = f"{s3_parent_config.instance}/{s3_parent_config.directory}" - if export_config.prefix: - export_path += f"/{export_config.prefix}" - - log_s3_export_command( - bucket=s3_parent_config.bucket, - export_path=export_path, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, + try: + # Collect file paths from previous steps + kpis_file = None + csv_file = None + ai_data_dir = None + analysis_file = None + + # Get KPI JSON file from artifacts_to_kpis step + artifacts_to_kpis_step = self._get_step("artifacts_to_kpis") + if ( + artifacts_to_kpis_step + and artifacts_to_kpis_step.get("status") == "success" + and artifacts_to_kpis_step.get("output_file") + ): + stored_output_file = artifacts_to_kpis_step["output_file"] + kpis_file = env.ARTIFACT_DIR / stored_output_file + logger.info( + f"S3 export: env.ARTIFACT_DIR={env.ARTIFACT_DIR}, stored_output_file={stored_output_file}, kpis_file={kpis_file}" ) - result = s3_export_entrypoint( - self.config, - output_dir, - kpis_file=kpis_file, - csv_file=csv_file, - ai_data_dir=ai_data_dir, - analysis_file=analysis_file, + # Get CSV file from kpis_to_csv step + kpis_to_csv_step = self._get_step("kpis_to_csv") + if ( + kpis_to_csv_step + and kpis_to_csv_step.get("status") == "success" + and kpis_to_csv_step.get("output_file") + ): + csv_file = env.ARTIFACT_DIR / kpis_to_csv_step["output_file"] + + # Get AI data directory from artifacts_to_ai_data step + ai_data_step = self._get_step("artifacts_to_ai_data") + + logger.info(f"AI data step found:\n{json.dumps(ai_data_step, indent=2)}") + if ( + ai_data_step + and ai_data_step.get("status") == "success" + and ai_data_step.get("ai_data_dir") + ): + ai_data_dir = env.ARTIFACT_DIR / ai_data_step["ai_data_dir"] + logger.info(f"AI data directory set to: {ai_data_dir}") + else: + logger.warning( + f"AI data directory not found - step status: {ai_data_step.get('status') if ai_data_step else 'step not found'}, ai_data_dir: {ai_data_step.get('ai_data_dir') if ai_data_step else 'N/A'}" ) - self._add_step("s3_export", result) - - if result.get("status") == "success": - # Format status for better readability - duration = result.get("duration", 0) - uploaded_files = result.get("uploaded_files", 0) - failed_files = result.get("failed_files", 0) - total_files = result.get("total_files", 0) - s3_path = result.get("exported_path", "") - - logger.info( - f"S3 export completed successfully in {duration}s: " - f"{uploaded_files}/{total_files} files uploaded" - f"{f' ({failed_files} failed)' if failed_files > 0 else ''} " - f"to {s3_path}" - ) - else: - self._check_step_result_and_set_failure("s3_export", result) - - except Exception as e: - error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" - - # Log programming errors more prominently - if isinstance(e, (AttributeError, NameError, TypeError)): - logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") - logger.exception("Full traceback for programming error:") - else: - logger.error(error_msg) - logger.exception("Full traceback:") + # Get analysis file from analyse_kpis step + analyze_step = self._get_step("analyse_kpis") + if ( + analyze_step + and analyze_step.get("status") in ("success", "warning") + and analyze_step.get("output_file") + ): + analysis_file = env.ARTIFACT_DIR / analyze_step["output_file"] + + # Create temporary status file for subprocess communication + status_file = output_dir / "s3_export_status.yaml" + + # Build CLI command + command = build_s3_export_command( + config=self.config, + status_file=status_file, + kpis_file=kpis_file, + csv_file=csv_file, + ai_data_dir=ai_data_dir, + analysis_file=analysis_file, + ) + + # Execute command using generic function + result, status_data, log_file = _execute_caliper_command( + command=command, + step_name="caliper s3 export", + status_file=status_file, + step_logs_dir=self.step_logs_dir, + ) + + # Clean up temporary status file + try: + status_file.unlink() + except FileNotFoundError: + pass + + # Convert to expected format + if status_data.get("success"): + step_result = { + "status": "success", + "exported_path": status_data.get("exported_path", ""), + "uploaded_files": status_data.get("uploaded_files", 0), + "total_files": status_data.get("total_files", 0), + "completed_at": time.time(), + } + + # Format status for better readability + uploaded_files = step_result.get("uploaded_files", 0) + total_files = step_result.get("total_files", 0) + s3_path = step_result.get("exported_path", "") + + logger.info( + f"S3 export completed successfully: " + f"{uploaded_files}/{total_files} files uploaded to {s3_path}" + ) + else: step_result = { "status": "failed", - "error": error_msg, - "exception_type": type(e).__name__, + "error": status_data.get("error", "Unknown error"), "completed_at": time.time(), } - self._add_step("s3_export", step_result) self._check_step_result_and_set_failure("s3_export", step_result) + self._add_step("s3_export", step_result, log_file) + + except Exception as e: + error_msg = f"S3 export step failed with exception: {type(e).__name__}: {str(e)}" + + # Log programming errors more prominently + if isinstance(e, AttributeError | NameError | TypeError): + logger.error(f"CRITICAL PROGRAMMING ERROR in S3 export: {error_msg}") + logger.exception("Full traceback for programming error:") + else: + logger.error(error_msg) + logger.exception("Full traceback:") + + step_result = { + "status": "failed", + "error": error_msg, + "exception_type": type(e).__name__, + "completed_at": time.time(), + } + self._add_step("s3_export", step_result, None) + self._check_step_result_and_set_failure("s3_export", step_result) + def _compute_final_status(self) -> str: """Compute the final postprocessing status.""" # Debug logging to identify what's causing failures @@ -1307,7 +1421,6 @@ def _compute_final_status(self) -> str: logger.info(f" s3_import_failed: {self.s3_import_failed}") logger.info(f" analyze_failed: {self.analyze_failed}") logger.info(f" s3_export_failed: {self.s3_export_failed}") - logger.info(f" analyze_failed: {self.analyze_failed}") final_status = compute_final_postprocess_status( test_outcome=self.test_outcome, diff --git a/projects/caliper/orchestration/postprocess_config.py b/projects/caliper/orchestration/postprocess_config.py index 4d1ca6815..13f789a8a 100644 --- a/projects/caliper/orchestration/postprocess_config.py +++ b/projects/caliper/orchestration/postprocess_config.py @@ -115,6 +115,10 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False + current_kpis: str | None = Field( + default="kpis.json", + description="Current KPI file path (relative to postprocess output dir unless absolute).", + ) historical_kpis: str | None = Field( default=None, description="Directory containing historical KPI JSON files (relative → postprocess output dir unless absolute).", @@ -125,11 +129,16 @@ class CaliperOrchestrationAnalyzeSection(BaseModel): ) @model_validator(mode="after") - def _historical_kpis_when_enabled(self) -> Self: - if self.enabled and not (self.historical_kpis and str(self.historical_kpis).strip()): - raise ValueError( - "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." - ) + def _validate_when_enabled(self) -> Self: + if self.enabled: + if not (self.current_kpis and str(self.current_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty current_kpis file path." + ) + if not (self.historical_kpis and str(self.historical_kpis).strip()): + raise ValueError( + "caliper.postprocess.analyze.enabled requires non-empty historical_kpis directory." + ) return self @@ -167,10 +176,6 @@ class CaliperOrchestrationS3ExportSection(BaseModel): model_config = ConfigDict(extra="forbid") enabled: bool = False - prefix: str | None = Field( - default=None, - description="S3 key prefix/folder path (optional).", - ) upload_id: str | None = Field( default=None, description="Custom upload identifier; uses timestamp (YY-MM-DD_HHMMSS) if null.", @@ -225,10 +230,6 @@ class CaliperOrchestrationS3Section(BaseModel): default=None, description="Directory identifier for S3 organization (required when import or export is enabled).", ) - prefix: str | None = Field( - default=None, - description="S3 prefix for organizing uploads and imports.", - ) vault: CaliperOrchestrationS3VaultSection = Field( default_factory=CaliperOrchestrationS3VaultSection, description="Vault configuration for AWS credentials.", diff --git a/projects/caliper/orchestration/postprocess_logging.py b/projects/caliper/orchestration/postprocess_logging.py new file mode 100644 index 000000000..fd4e15daa --- /dev/null +++ b/projects/caliper/orchestration/postprocess_logging.py @@ -0,0 +1,172 @@ +""" +Command logging and execution utilities for Caliper postprocessing. + +Handles formatting commands for logs, writing log headers/footers, and executing +commands with proper logging and status handling. +""" + +from __future__ import annotations + +import logging +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _format_command_for_log(command: list[str]) -> str: + """Format command for log file display with only --arguments on new lines. + + Args: + command: Command arguments list + + Returns: + Formatted command string for log file + """ + if not command: + return "" + + # Start with base command parts until we hit options + lines = [] + base_parts = [] + i = 0 + + # Collect all non-flag arguments into base_parts + while i < len(command) and not command[i].startswith("--"): + base_parts.append(command[i]) + i += 1 + + if base_parts: + lines.append(" ".join(base_parts)) + + # Handle --flag arguments on separate lines + while i < len(command): + if command[i].startswith("--"): + if i + 1 < len(command) and not command[i + 1].startswith("--"): + # Flag with value + lines.append(f"{command[i]} {command[i + 1]}") + i += 2 + else: + # Boolean flag + lines.append(command[i]) + i += 1 + else: + # Standalone argument + lines.append(command[i]) + i += 1 + + return " \\\n ".join(lines) + + +def _write_step_header_to_log_file(log_file: Path, command: list[str], step_name: str) -> None: + """Write step header to log file. + + Args: + log_file: Path to log file + command: Command that will be executed + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + formatted_command = _format_command_for_log(command) + header = f"""{"=" * 60} +🚀 STARTING {step_name} (fork/exec) +📄 Command: + {formatted_command} +{"=" * 60} + +""" + with open(log_file, "w", encoding="utf-8") as f: + f.write(header) + + +def _write_step_footer_to_log_file( + log_file: Path, result: subprocess.CompletedProcess, step_name: str +) -> None: + """Write step footer to log file. + + Args: + log_file: Path to log file + result: Subprocess result + step_name: Name of the step (e.g., "CALIPER PARSE") + """ + footer = f""" + +{"=" * 60} +🏁 {step_name} COMPLETED +📊 Exit code: {result.returncode} +📄 Log file: {log_file} +{"=" * 60} +""" + with open(log_file, "a", encoding="utf-8") as f: + f.write(footer) + + +def _handle_caliper_output_and_completion_with_header( + result: subprocess.CompletedProcess, + log_file: Path, + status_file: Path, + step_name: str, +) -> dict: + """Handle Caliper output writing, status parsing, and log completion banner. + + This version appends to the log file instead of overwriting it, since we've + already written a header. + + Args: + result: Subprocess result from caliper execution + log_file: Path to write the step log file + status_file: Path to the status file to read and parse + step_name: Name of the Caliper step (e.g., "PARSE", "VISUALIZE") + + Returns: + Parsed status data dictionary + """ + import yaml + + # Append output to log file (don't overwrite since we've written header) + with open(log_file, "a", encoding="utf-8") as log_f: + if result.stdout: + log_f.write(result.stdout) + + # Also display output in main orchestration logs + if result.stdout: + prefix = step_name.upper() + for line in result.stdout.splitlines(): + logger.info("%s: %s", prefix, line) + + # Log banner indicating Caliper execution completed + logger.info("=" * 60) + logger.info("🏁 CALIPER %s COMPLETED", step_name.upper()) + logger.info("📊 Exit code: %s", result.returncode) + logger.info("📄 Log file: %s", log_file) + + # Read and parse status file + try: + with open(status_file, encoding="utf-8") as f: + status_data = yaml.safe_load(f) + # Handle case where YAML file is empty or invalid + if status_data is None: + status_data = { + "success": False, + "error": "Status file is empty or contains no valid YAML data", + } + # Normalize non-mapping payloads (lists, strings, integers) into failure dict + elif not isinstance(status_data, dict): + status_data = { + "success": False, + "error": f"Status file contains non-mapping data: {type(status_data).__name__}", + "raw_data": status_data, + } + except Exception as e: + status_data = {"success": False, "error": f"Failed to read status file: {e}"} + + # Log status file content for visibility + logger.info("📄 Status file content:") + if isinstance(status_data, dict): + for key, value in status_data.items(): + logger.info(" %s: %s", key, value) + else: + logger.info(" %s", status_data) + + logger.info("=" * 60) + + return status_data diff --git a/projects/caliper/orchestration/replot.py b/projects/caliper/orchestration/replot.py index 8ff5f6f69..fd2132327 100644 --- a/projects/caliper/orchestration/replot.py +++ b/projects/caliper/orchestration/replot.py @@ -13,6 +13,7 @@ from projects.caliper.orchestration.postprocess import run_postprocess_from_orchestration_config from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome +from projects.core.library import env from projects.core.library import vault as vault_lib logger = logging.getLogger(__name__) @@ -186,7 +187,7 @@ def run_replot_from_orchestration_config( if skip_download: keep_replot_dir = True - logger.info("No replot URL provided, skipping artifact download") + logger.warning("No replot URL provided, skipping artifact download") logger.info("Automatically setting keep_replot_dir=True to preserve existing files") logger.info(f"Download directory: {replot_download_dir}") logger.info(f"Output directory: {artifact_directory}") @@ -304,15 +305,14 @@ def run_replot_from_orchestration_config( if artifact_directory.exists(): logger.info("Running post-processing...") - visualize_output_dir = artifact_directory / "postprocess_output" - visualize_output_dir.mkdir(parents=True, exist_ok=True) - - postprocess_result = run_postprocess_from_orchestration_config( - postprocess_config_raw=postprocess_config or {}, - artifacts_dir=artifact_directory, - visualize_output_dir=visualize_output_dir, - test_outcome=TestPhaseOutcome("SUCCESS"), - ) + with env.TempArtifactDir(artifact_directory / "postprocess_output"): + # Let Caliper handle output directory resolution from config + postprocess_result = run_postprocess_from_orchestration_config( + postprocess_config_raw=postprocess_config or {}, + artifacts_dir=artifact_directory, + visualize_output_dir=None, # Let Caliper resolve from config + test_outcome=TestPhaseOutcome("SUCCESS"), + ) else: logger.info("Artifacts directory not found") postprocess_result = { diff --git a/projects/caliper/orchestration/step_logging.py b/projects/caliper/orchestration/step_logging.py index 3aeae431b..0febd3ecb 100644 --- a/projects/caliper/orchestration/step_logging.py +++ b/projects/caliper/orchestration/step_logging.py @@ -105,9 +105,12 @@ def step_logging_indexed( """ Context manager for step-specific logging with explicit step index. - Captures all logging output during the context and writes it to a dedicated + Captures ALL logging output during the context and writes it to a dedicated file named with the pattern: {step_index:03d}__{step_name}.log + This context manager temporarily configures the entire logging system to + ensure that ALL log messages from any logger get captured to the step log file. + Args: step_name: Name of the step (e.g., "caliper_parse", "caliper_visualize") step_index: Zero-based index of the step for ordering (e.g., 0, 1, 2...) @@ -121,9 +124,6 @@ def step_logging_indexed( logger.info("This will go to 000__caliper_parse.log") run_parse_operations() """ - # Ensure the global step handler is attached - _ensure_step_handler() - # Create the log file path log_filename = f"{step_index:03d}__{step_name}.log" log_file = output_dir / log_filename @@ -138,16 +138,55 @@ def step_logging_indexed( # Use same format as console output (no timestamp prefix, just the message) file_handler.setFormatter(logging.Formatter("%(message)s")) - # Store the file handler in thread-local storage - _step_local_handlers.file_handler = file_handler + # Get the root logger to capture everything + root_logger = logging.getLogger() + + # Save original state to restore later + original_handlers = root_logger.handlers.copy() + + # Note: We now use direct root logger capture instead of the StepLocalHandler mechanism + # This provides more reliable and comprehensive log capture try: + # Add our file handler directly to the root logger + root_logger.addHandler(file_handler) + + # Save propagation states and ensure loggers propagate to root + saved_logger_states = {} + + # Get all existing loggers from the logger registry + for logger_name in logging.getLogger().manager.loggerDict: + existing_logger = logging.getLogger(logger_name) + # Save original state (only propagate, don't touch levels) + saved_logger_states[logger_name] = { + "propagate": existing_logger.propagate, + } + # Only ensure propagation, don't change levels + existing_logger.propagate = True + yield log_file + finally: - # Clean up: close and remove the thread-local handler - if hasattr(_step_local_handlers, "file_handler"): - _step_local_handlers.file_handler.close() - del _step_local_handlers.file_handler + # Remove our file handler from root logger + if file_handler in root_logger.handlers: + root_logger.removeHandler(file_handler) + + # Restore original handlers completely + # (Remove any handlers that were added, restore original list) + root_logger.handlers.clear() + root_logger.handlers.extend(original_handlers) + + # Restore original logger propagation states + for logger_name, state in saved_logger_states.items(): + try: + existing_logger = logging.getLogger(logger_name) + existing_logger.propagate = state["propagate"] + except Exception: + # Ignore errors when restoring logger states + pass + + # Close the file handler + file_handler.close() @contextmanager @@ -392,36 +431,16 @@ def log_s3_import_command( bucket: str, prefix: str, output_dir: Path, - include_kpis_json: bool = False, - include_kpis_csv: bool = False, - include_ai_data: bool = False, - max_downloads: int = 50, ) -> None: """Log the CLI command to reproduce the S3 import step.""" - command_parts = [ + command = ( f'caliper s3-import --bucket "{bucket}" --prefix "{prefix}" --output-dir "{output_dir}"' - ] - - # Add include flags if they are True (since default is False) - if include_kpis_json: - command_parts.append("--include-kpis-json") - if include_kpis_csv: - command_parts.append("--include-kpis-csv") - if include_ai_data: - command_parts.append("--include-ai-data") - if max_downloads != 50: - command_parts.append(f"--max-downloads {max_downloads}") - - command = " ".join(command_parts) + ) step_args = { "bucket": bucket, "prefix": prefix, "output_dir": str(output_dir), - "include_kpis_json": include_kpis_json, - "include_kpis_csv": include_kpis_csv, - "include_ai_data": include_ai_data, - "max_downloads": max_downloads, } _log_command_banner( @@ -435,28 +454,13 @@ def log_s3_import_command( def log_s3_export_command( bucket: str, export_path: str, - kpis_file: Path | None = None, - csv_file: Path | None = None, - ai_data_dir: Path | None = None, - analysis_file: Path | None = None, + from_dir: Path, + include_csv: bool, + include_kpis_json: bool, + include_ai_data: bool, ) -> None: """Log the CLI command to reproduce the S3 export step.""" - command_parts = [f'caliper s3-export --bucket "{bucket}"'] - - # Add file paths - if kpis_file: - command_parts.append(f'--kpis-file "{kpis_file}"') - if csv_file: - command_parts.append(f'--csv-file "{csv_file}"') - if ai_data_dir: - command_parts.append(f'--ai-data-dir "{ai_data_dir}"') - if analysis_file: - command_parts.append(f'--analysis-file "{analysis_file}"') - - # Parse export_path once and use for both command and step_args - instance = None - directory = None - prefix = None + command_parts = [f'caliper s3-export --from-dir "{from_dir}" --bucket "{bucket}"'] if export_path: # Parse the export path to get prefix, instance, directory @@ -470,24 +474,19 @@ def log_s3_export_command( if prefix: command_parts.append(f'--prefix "{prefix}"') + # Note: Include flags use default values (--include-csv, --include-kpis-json, --include-ai-data all default to True) + command = " ".join(command_parts) step_args = { "bucket": bucket, - "kpis_file": str(kpis_file) if kpis_file else None, - "csv_file": str(csv_file) if csv_file else None, - "ai_data_dir": str(ai_data_dir) if ai_data_dir else None, - "analysis_file": str(analysis_file) if analysis_file else None, + "export_path": export_path, + "from_dir": str(from_dir), + "include_csv": include_csv, + "include_kpis_json": include_kpis_json, + "include_ai_data": include_ai_data, } - # Add command line parameters to step_args (same values used in command) - if instance: - step_args["instance"] = instance - if directory: - step_args["directory"] = directory - if prefix: - step_args["prefix"] = prefix - _log_command_banner( "caliper_s3_export", command, @@ -496,30 +495,37 @@ def log_s3_export_command( ) -def log_analyze_command( - base_dir: Path, - plugin_module: str, - current_kpis_path: Path | None = None, - historical_kpis_dir: Path | None = None, - output_path: Path | None = None, +def log_analyse_kpis_command( + current_kpi_file: Path, + historical_dir: Path, + output_file: Path, ) -> None: - """Log the CLI command to reproduce the analyze step.""" - - command = f'caliper kpi analyze --current "{current_kpis_path}" --baseline-dir "{historical_kpis_dir}" --output "{output_path}" --plugin "{plugin_module}"' + """Log the CLI command to reproduce the KPI analysis step.""" + command = f'caliper analyse-kpis --current "{current_kpi_file}" --historical-dir "{historical_dir}" --output "{output_file}"' step_args = { - "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, - "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, - "output_path": str(output_path) if output_path else None, - "plugin_module": plugin_module, + "current_kpi_file": str(current_kpi_file), + "historical_dir": str(historical_dir), + "output_file": str(output_file), } _log_command_banner( - "caliper_analyze_kpis", + "caliper_analyse_kpis", command, - "Analyze current KPIs against historical data for regression detection", + "Analyze current KPIs against historical data", step_args, ) + + +def log_analyze_command( + base_dir: Path, + plugin_module: str, + current_kpis_path: Path | None = None, + baseline_file: Path | None = None, + historical_kpis_dir: Path | None = None, + output_path: Path | None = None, +) -> None: + """Log the CLI command to reproduce the analyze step.""" # Use the correct command structure: caliper kpi analyze command = "caliper kpi analyze" @@ -539,15 +545,18 @@ def log_analyze_command( command += f" --plugin {plugin_module}" # Add helpful context - command += "\n# Analyzes KPI for regressions" + command += "\n# Analyzes hierarchical KPI format for regressions" command += "\n# The tool will automatically find the most recent kpis.json file in the baseline directory" command += "\n# To re-run this analysis step independently, use the command above" command += f"\n# Plugin: {plugin_module}" command += f"\n# Base dir: {base_dir}" + if baseline_file: + command += f"\n# Most recent baseline file found: {baseline_file}" step_args = { "current_kpis_path": str(current_kpis_path) if current_kpis_path else None, "historical_kpis_dir": str(historical_kpis_dir) if historical_kpis_dir else None, + "baseline_file_selected": str(baseline_file) if baseline_file else None, "output_path": str(output_path) if output_path else None, "plugin_module": plugin_module, "base_dir": str(base_dir), diff --git a/projects/caliper/postprocess/helpers/visualization_utils.py b/projects/caliper/postprocess/helpers/visualization_utils.py index 68d45c361..a6e6f4c2b 100644 --- a/projects/caliper/postprocess/helpers/visualization_utils.py +++ b/projects/caliper/postprocess/helpers/visualization_utils.py @@ -3,8 +3,11 @@ from __future__ import annotations import base64 +import logging from pathlib import Path +logger = logging.getLogger(__name__) + def save_figure( fig, @@ -12,8 +15,8 @@ def save_figure( filename: str, as_image: bool = True, report_number: int | None = None, - width: int = 800, - height: int = 500, + width: int = 1200, + height: int = 650, ) -> str | None: """ Save a plotly figure as either an image or HTML file with optional report numbering. @@ -38,135 +41,34 @@ def save_figure( final_filename = filename if as_image: - print(f"💾 Saving {final_filename} as PNG image...") + logger.info(f"Saving {final_filename} as PNG image...") output_file = output_dir / f"{final_filename}.png" - fig.write_image(output_file, width=width, height=height, scale=2) + fig.write_image(output_file, width=width, height=height) else: - print(f"💾 Saving {final_filename} as full-page interactive HTML...") + logger.info(f"Saving {final_filename} as full-page interactive HTML...") output_file = output_dir / f"{final_filename}.html" - # Configure the figure for full-page display + # Configure figure for full-screen responsive behavior fig.update_layout( autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - ) - - # Save with full-page configuration - fig.write_html( - output_file, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, + width=None, # Remove any fixed width + height=None, # Remove any fixed height ) + fig.write_html(output_file) - # Make it full page - _make_html_full_page(output_file) - - print(f"✅ {final_filename} saved successfully") + logger.info(f"{final_filename} saved successfully") return str(output_file) except Exception as e: final_filename = ( f"report_{report_number:02d}_{filename}" if report_number is not None else filename ) - print(f"❌ Failed to save figure {final_filename}: {e}") + logger.error(f"Failed to save figure {final_filename}: {e}") return None -def _make_html_full_page(html_file_path: str) -> None: - """ - Modify an HTML file to make the plot full page by adding custom CSS. - - Args: - html_file_path: Path to the HTML file to modify - """ - try: - with open(html_file_path, encoding="utf-8") as f: - content = f.read() - - # Insert full-page CSS styles - full_page_css = """ - -""" - - # Insert the CSS before the closing tag - content = content.replace("", f"{full_page_css}") - - with open(html_file_path, "w", encoding="utf-8") as f: - f.write(content) - - except Exception as e: - print(f"⚠️ Warning: Failed to make HTML full page: {e}") - - -def write_full_page_html(fig, output_file_path: str, title: str = "Plot") -> bool: - """ - Save a Plotly figure as a full-page HTML file. - - Args: - fig: Plotly figure object - output_file_path: Path where to save the HTML file - title: Title for the HTML page - - Returns: - True if successful, False otherwise - """ - try: - # Configure the figure for full-page display - fig.update_layout( - autosize=True, - margin=dict(l=20, r=20, t=60, b=40), - title_text=title if title != "Plot" else None, - ) - - # Save with full-page configuration - fig.write_html( - output_file_path, - include_plotlyjs="cdn", - config={ - "displayModeBar": True, - "displaylogo": False, - "modeBarButtonsToRemove": ["pan2d", "lasso2d"], - "responsive": True, - }, - div_id="plotly-div", - full_html=True, - include_mathjax=False, - ) - - # Make it full page - _make_html_full_page(output_file_path) - return True - - except Exception as e: - print(f"❌ Failed to save full-page HTML: {e}") - return False - - def figure_to_base64( - fig, width: int = 800, height: int = 500, plot_name: str = "plot" + fig, width: int = 1200, height: int = 650, plot_name: str = "plot" ) -> str | None: """ Convert a plotly figure to base64-encoded PNG for embedding in HTML. @@ -181,20 +83,20 @@ def figure_to_base64( Base64-encoded image string with data URI prefix, or None if failed """ try: - print(f"🖼️ Converting {plot_name} to high-quality PNG ({width}x{height})...") + logger.info(f"Converting {plot_name} to high-quality PNG ({width}x{height})...") # Convert figure to PNG bytes - img_bytes = fig.to_image(format="png", width=width, height=height, scale=2) + img_bytes = fig.to_image(format="png", width=width, height=height) - print(f"📦 Encoding {plot_name} as base64 for HTML embedding...") + logger.info(f"Encoding {plot_name} as base64 for HTML embedding...") # Encode as base64 img_base64 = base64.b64encode(img_bytes).decode() - print(f"✅ {plot_name} image ready ({len(img_base64) // 1024}KB)") + logger.info(f"{plot_name} image ready ({len(img_base64) // 1024}KB)") return f"data:image/png;base64,{img_base64}" except Exception as e: - print(f"❌ Failed to convert {plot_name} to base64: {e}") + logger.error(f"Failed to convert {plot_name} to base64: {e}") return None @@ -265,3 +167,28 @@ def create_report_title_display(base_title: str, report_number: int | None = Non return f"Report {report_number:02d}: {base_title}" else: return base_title + + +def write_full_page_html(fig, output_path: str, title: str) -> bool: + """ + Write a plotly figure as a full-page HTML file. + + Args: + fig: Plotly figure object + output_path: Full path to the output HTML file + title: Title for the HTML document (currently unused but kept for compatibility) + + Returns: + True if successful, False otherwise + """ + try: + output_file = Path(output_path) + output_dir = output_file.parent + filename = output_file.stem # Get filename without extension + + # Use save_figure to write HTML + result = save_figure(fig, output_dir, filename, as_image=False) + return result is not None + except Exception as e: + logger.error(f"Failed to write full-page HTML to {output_path}: {e}") + return False diff --git a/projects/caliper/tests/test_kpi_analyze.py b/projects/caliper/tests/test_kpi_analyze.py new file mode 100644 index 000000000..2ca06e4de --- /dev/null +++ b/projects/caliper/tests/test_kpi_analyze.py @@ -0,0 +1,442 @@ +"""Tests for KPI regression analysis (analyze.py).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from projects.caliper.engine.kpi.analyze import ( + AnalysisConfig, + _build_baseline_index, + _extract_kpi_records_from_hierarchical, + _match_key, + _run_regression_test, + run_kpi_analysis, +) + + +def _make_hierarchical_kpi( + tests: list[dict], +) -> dict: + """Helper to build a schema_version=2 hierarchical KPI doc.""" + return {"schema_version": "2", "tests": tests} + + +def _make_test_entry( + run_id: str, + labels: dict, + kpis: list[dict], +) -> dict: + return { + "run_id": run_id, + "labels": labels, + "metadata": {"timestamp": "2025-01-01T00:00:00Z"}, + "kpis": kpis, + } + + +def _make_kpi(kpi_id: str, value, unit: str = "tokens/s", higher_is_better: bool = True) -> dict: + return {"id": kpi_id, "value": value, "unit": unit, "higher_is_better": higher_is_better} + + +class TestMatchKey: + def test_basic_match_key(self): + labels = {"platform": "A100", "version": "1.0", "os": "linux"} + key = _match_key(labels, ignored_keys=["os"], comparison_keys=["version"]) + assert ("os", "linux") not in key + assert ("version", "1.0") not in key + assert ("platform", "A100") in key + + def test_empty_config(self): + labels = {"a": "1", "b": "2"} + key = _match_key(labels, ignored_keys=[], comparison_keys=[]) + assert key == tuple(sorted([("a", "1"), ("b", "2")])) + + +class TestExtractRecords: + def test_extracts_flat_records(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("latency", 0.5, unit="s", higher_is_better=False), + ], + ), + ] + ) + records = _extract_kpi_records_from_hierarchical(data) + assert len(records) == 2 + assert records[0]["kpi_id"] == "throughput" + assert records[0]["labels"] == {"platform": "A100"} + assert records[1]["value"] == 0.5 + + def test_empty_tests(self): + data = _make_hierarchical_kpi([]) + assert _extract_kpi_records_from_hierarchical(data) == [] + + +class TestBuildBaselineIndex: + def test_indexes_by_kpi_and_match_key(self): + data = _make_hierarchical_kpi( + [ + _make_test_entry( + "run1", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + _make_test_entry( + "run2", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + config = AnalysisConfig(comparison_keys=["version"]) + baseline_data = {Path("/fake/kpis.json"): data} + index = _build_baseline_index(baseline_data, config) + + mk = _match_key({"platform": "A100"}, [], ["version"]) + key = ("throughput", mk) + assert key in index + assert len(index[key]) == 2 + + +class TestRegressionTest: + def test_no_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 100.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 95.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + assert result.relative_change > 0 + + def test_regression_higher_is_better(self): + current = {"kpi_id": "throughput", "value": 80.0, "higher_is_better": True, "labels": {}} + baselines = [{"value": 100.0}, {"value": 100.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change < -0.1 + + def test_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 1.5, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}, {"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert result.regression + assert result.relative_change > 0.1 + + def test_no_regression_lower_is_better(self): + current = {"kpi_id": "latency", "value": 0.9, "higher_is_better": False, "labels": {}} + baselines = [{"value": 1.0}] + config = AnalysisConfig(max_relative_regression=0.1) + result = _run_regression_test(current, baselines, config) + assert not result.regression + + +class TestEndToEnd: + """Full end-to-end test of run_kpi_analysis.""" + + def _write_hierarchical_kpi(self, path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(data, f) + + def test_pass_no_regression(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current KPIs: throughput=100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Historical: throughput=95, 98 (within 10% threshold) + baseline1 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_1", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + baseline2 = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run_2", + {"platform": "A100"}, + [ + _make_kpi("throughput", 98.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline1) + self._write_hierarchical_kpi(historical_dir / "run2" / "kpis.json", baseline2) + + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert test_status["exit_code"] == 0 + assert output_file.exists() + + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "PASS" + assert report["overall"]["verdict"] == "PASS" + assert report["overall"]["regression_count"] == 0 + assert report["tested"]["total_kpis"] == 1 + + def test_regression_detected(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: throughput dropped from 100 to 70 (30% regression) + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "current_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 70.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old_run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert test_status["exit_code"] == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["analysis"]["status"] == "REGRESSION_DETECTED" + assert report["overall"]["regression_count"] == 1 + assert report["results"][0]["verdict"] == "REGRESSION" + assert report["results"][0]["relative_change_pct"] == pytest.approx(-30.0) + + def test_no_historical_data(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + historical_dir.mkdir(parents=True) + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert test_status["exit_code"] == 2 + with open(output_file) as f: + report = yaml.safe_load(f) + assert report["overall"]["verdict"] == "NO_BASELINE" + + def test_mixed_regression_and_pass(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 2.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0, higher_is_better=True), + _make_kpi("latency", 1.0, unit="s", higher_is_better=False), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert test_status["exit_code"] == 3 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["overall"]["regression_count"] == 1 + latency_result = next(r for r in report["results"] if r["kpi_id"] == "latency") + assert latency_result["verdict"] == "REGRESSION" + throughput_result = next(r for r in report["results"] if r["kpi_id"] == "throughput") + assert throughput_result["verdict"] == "PASS" + + def test_skips_non_scalar_kpis(self, tmp_path): + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100"}, + [ + _make_kpi("throughput", 100.0), + _make_kpi("curve_data", [1.0, 2.0, 3.0]), # non-scalar + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + assert test_status["exit_code"] == 0 + with open(output_file) as f: + report = yaml.safe_load(f) + + assert report["tested"]["skipped"] == 1 + assert report["tested"]["total_kpis"] == 1 + + def test_comparison_keys_separate_baselines(self, tmp_path): + """Records that differ on comparison_keys should still be matched.""" + current_dir = tmp_path / "current" + historical_dir = tmp_path / "historical" + output_file = tmp_path / "report.yaml" + + # Current: version=2.0 on platform=A100 + current = _make_hierarchical_kpi( + [ + _make_test_entry( + "run", + {"platform": "A100", "version": "2.0"}, + [ + _make_kpi("throughput", 100.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(current_dir / "kpis.json", current) + + # Baseline: version=1.0 on platform=A100 (different comparison key value) + baseline = _make_hierarchical_kpi( + [ + _make_test_entry( + "old", + {"platform": "A100", "version": "1.0"}, + [ + _make_kpi("throughput", 95.0), + ], + ), + ] + ) + self._write_hierarchical_kpi(historical_dir / "run1" / "kpis.json", baseline) + + # With version as comparison_key, both match on platform=A100 + # (plugin would expose this config; here we test the core logic directly) + test_status = run_kpi_analysis( + current_kpi_file=current_dir / "kpis.json", + historical_data_dir=historical_dir, + output_file=output_file, + plugin_module="nonexistent_plugin", + ) + + # Without plugin config, default AnalysisConfig has no comparison_keys, + # so version must also match → no baselines found → skip + assert test_status["exit_code"] in (0, 2) + with open(output_file) as f: + report = yaml.safe_load(f) + # The version label differs, so with empty comparison_keys they don't match + assert report["tested"]["total_kpis"] == 0 or report["overall"]["verdict"] == "NO_BASELINE" diff --git a/projects/cluster/toolbox/capture_prometheus/README.md b/projects/cluster/toolbox/capture_prometheus/README.md new file mode 100644 index 000000000..63c8d09ce --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/README.md @@ -0,0 +1,79 @@ +# capture_prometheus + +Extracts cluster Prometheus metrics for a specific time window and saves them as a compressed OpenMetrics formated file. The output can later be imported into a local Prometheus instance for offline querying of any metric. + +## How it works + +1. **Symlink trick** — Creates a temporary directory on the Prometheus PVC containing only symlinks to `wal/` and `chunks_head/`. This limits `promtool` to scanning the recent in-memory data (~1-2 GB) instead of the full TSDB (which can be 20+ GB of persistent blocks). +2. **Time-filtered dump** — Runs `promtool tsdb dump-openmetrics --min-time=X --max-time=Y` inside the Prometheus container against the temp directory. Only samples within the requested window are emitted. +3. **Compress and copy** — The output is gzipped on the pod, then copied to the specified output directory via `oc cp`. +4. **Cleanup** — Removes the temp directory and compressed file from the pod. + + + +## Limitations + +- **Max test duration: 2 hours.** The WAL + chunks_head holds approximately the last 2 hours of un-compacted data. Tests longer than 2 hours will fail with an error. Support for including persistent blocks that overlap with the test window can be added later. +- **Data granularity:** The dump includes all metrics scraped by cluster monitoring (typically every 30s). For a 20-minute test this produces ~24M data points / ~130MB compressed. + + + +## Usage + + + +### Standalone (CLI) + +```bash +./bin/run_toolbox cluster capture_prometheus \ + "2026-07-26T10:00:00+00:00" \ + "2026-07-26T10:20:00+00:00" \ + /path/to/output +``` + + + +### From orchestration + +```python +from projects.cluster.toolbox.capture_prometheus import main as capture_prom + +prometheus_dir = env.ARTIFACT_DIR / "prometheus" +prometheus_dir.mkdir(parents=True, exist_ok=True) + +capture_prom.run( + start_time=test_start_time, + end_time=test_end_time, + output_dir=str(prometheus_dir), +) +``` + + + +## Querying the data locally + +To analyze the captured metrics offline: + +1. **Decompress** the archive to get a plain OpenMetrics text file. +2. **Import** the file into a Prometheus TSDB using `promtool tsdb create-blocks-from openmetrics`. +3. **Run a Prometheus container** (Docker, Podman, or any OCI runtime) with: + - The imported TSDB directory mounted at `/prometheus` + - A minimal config file (an empty `global: {}` is sufficient since no scraping is needed) + - Retention set high enough that the data won't be garbage-collected (e.g. `--storage.tsdb.retention.time=365d`) +4. **Open the Prometheus UI** and query any metric using the time range from `metadata.json`. + +The output file (`metrics.openmetrics.gz`) follows the standard [OpenMetrics](https://openmetrics.io/) format and is compatible with any tool that can ingest it (Prometheus, Grafana via Prometheus data source, etc.). + +## Parameters + + +| Parameter | Required | Default | Description | +| ------------- | -------- | ---------------------- | ------------------------------------------------ | +| `start_time` | yes | — | Start of capture window (ISO 8601, UTC) | +| `end_time` | yes | — | End of capture window (ISO 8601, UTC) | +| `output_dir` | yes | — | Directory to write `metrics.openmetrics.gz` into | +| `--pod-name` | no | `prometheus-k8s-0` | Prometheus pod name | +| `--namespace` | no | `openshift-monitoring` | Namespace of Prometheus | +| `--container` | no | `prometheus` | Container name in the pod | + + diff --git a/projects/cluster/toolbox/capture_prometheus/main.py b/projects/cluster/toolbox/capture_prometheus/main.py new file mode 100644 index 000000000..8fcd7949d --- /dev/null +++ b/projects/cluster/toolbox/capture_prometheus/main.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 + +""" +Capture Prometheus TSDB Toolbox + +Extracts cluster Prometheus metrics for a specific time window and saves them +as a compressed OpenMetrics archive. The output can later be imported into a +local Prometheus instance for offline querying. + +Can be run standalone: + ./bin/run_toolbox cluster capture_prometheus \\ + "2026-07-26T10:00:00+00:00" \\ + "2026-07-26T10:20:00+00:00" \\ + /path/to/output +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from pathlib import Path + +from projects.core.dsl import always, entrypoint, execute_tasks, task +from projects.core.dsl.utils.k8s import oc, oc_cp_from_pod, oc_exec + +logger = logging.getLogger("DSL") + +PROMETHEUS_NAMESPACE = "openshift-monitoring" +PROMETHEUS_POD = "prometheus-k8s-0" +PROMETHEUS_CONTAINER = "prometheus" +TSDB_PATH = "/prometheus" + + +@entrypoint +def run( + start_time: datetime, + end_time: datetime, + output_dir: str | Path, + *, + pod_name: str = PROMETHEUS_POD, + namespace: str = PROMETHEUS_NAMESPACE, + container: str = PROMETHEUS_CONTAINER, +) -> int: + """ + Capture Prometheus metrics for a specific time window. + + Extracts all metrics between start_time and end_time from the cluster's + Prometheus TSDB and writes them as a compressed OpenMetrics file to output_dir. + + Args: + start_time: Start of the capture window (UTC) + end_time: End of the capture window (UTC) + output_dir: Directory to write the metrics archive into + pod_name: Prometheus pod to extract from + namespace: Namespace where Prometheus runs + container: Container name within the Prometheus pod + """ + execute_tasks(locals()) + return 0 + + +def _pod_kwargs(ctx) -> dict: + """Return the common namespace/pod/container kwargs for oc_exec/oc_cp_from_pod.""" + return {"namespace": ctx.namespace, "pod": ctx.pod_name, "container": ctx.container} + + +def _parse_time(value) -> datetime: + """Parse a datetime from string (ISO format) or pass through if already datetime. + + Naive datetimes are assumed UTC. Aware datetimes are normalized to UTC. + """ + if isinstance(value, datetime): + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + dt = datetime.fromisoformat(str(value)) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +@task +def validate_parameters(args, ctx): + """Validate inputs and compute time bounds.""" + + ctx.start_time = _parse_time(args.start_time) + ctx.end_time = _parse_time(args.end_time) + + if ctx.end_time <= ctx.start_time: + raise ValueError("end_time must be after start_time") + + max_duration_seconds = 2 * 3600 + duration = (ctx.end_time - ctx.start_time).total_seconds() + if duration > max_duration_seconds: + raise ValueError( + f"Test duration ({int(duration)}s) exceeds the 2-hour WAL window. " + f"Tests longer than 2 hours require persistent block handling (not yet implemented)." + ) + + ctx.output_dir = Path(args.output_dir) + ctx.output_dir.mkdir(parents=True, exist_ok=True) + ctx.start_ms = int(ctx.start_time.timestamp() * 1000) + ctx.end_ms = int(ctx.end_time.timestamp() * 1000) + ctx.duration_seconds = int((ctx.end_time - ctx.start_time).total_seconds()) + + ctx.namespace = args.namespace + ctx.pod_name = args.pod_name + ctx.container = args.container + + return ( + f"Capture window: {ctx.start_time.isoformat()} → {ctx.end_time.isoformat()} " + f"({ctx.duration_seconds}s)" + ) + + +@task +def validate_prometheus_pod(args, ctx): + """Verify the Prometheus pod is running and accessible.""" + + result = oc( + "-n", + ctx.namespace, + "get", + "pod", + ctx.pod_name, + "-o", + "jsonpath={.status.phase}", + ) + + phase = result.stdout.strip() + if phase != "Running": + raise RuntimeError( + f"Prometheus pod {ctx.pod_name} is in phase '{phase}', expected 'Running'" + ) + + return f"Prometheus pod {ctx.pod_name} is running" + + +@task +def create_temp_tsdb_dir(args, ctx): + """Create a temp directory on the PVC with symlinks to WAL and chunks_head only. + + This limits promtool to scanning only recent in-memory data (~1-2 GB) + instead of the full TSDB which can be tens of GB. + """ + + ctx.temp_dir = f"{TSDB_PATH}/.capture-tmp-{ctx.start_ms}" + + oc_exec( + "sh", + "-c", + f"mkdir -p {ctx.temp_dir}" + f" && ln -sf {TSDB_PATH}/wal {ctx.temp_dir}/wal" + f" && ln -sf {TSDB_PATH}/chunks_head {ctx.temp_dir}/chunks_head", + **_pod_kwargs(ctx), + ) + + return f"Created temp TSDB dir at {ctx.temp_dir}" + + +@task +def dump_metrics(args, ctx): + """Run promtool tsdb dump-openmetrics with time filtering against the temp dir.""" + + ctx.remote_raw = f"{TSDB_PATH}/.capture-metrics-{ctx.start_ms}" + ctx.remote_output = f"{ctx.remote_raw}.gz" + + oc_exec( + "sh", + "-c", + f"promtool tsdb dump-openmetrics" + f" --min-time={ctx.start_ms} --max-time={ctx.end_ms}" + f" {ctx.temp_dir} > {ctx.remote_raw}" + f" && gzip -f {ctx.remote_raw}", + **_pod_kwargs(ctx), + ) + + size_result = oc_exec("stat", "-c", "%s", ctx.remote_output, check=False, **_pod_kwargs(ctx)) + + ctx.archive_size_bytes = int(size_result.stdout.strip()) if size_result.success else 0 + + return f"Dumped metrics ({ctx.archive_size_bytes} bytes compressed)" + + +@task +def copy_to_output(args, ctx): + """Copy the compressed metrics file from the pod to the output directory.""" + + ctx.output_file = ctx.output_dir / "metrics.openmetrics.gz" + + oc_cp_from_pod(ctx.remote_output, ctx.output_file, **_pod_kwargs(ctx)) + + return f"Copied metrics to {ctx.output_file}" + + +@always +@task +def cleanup_pod(args, ctx): + """Remove temp files from the Prometheus pod.""" + + temp_dir = getattr(ctx, "temp_dir", None) + remote_output = getattr(ctx, "remote_output", None) + remote_raw = getattr(ctx, "remote_raw", None) + kwargs = _pod_kwargs(ctx) + + for path in [temp_dir, remote_output, remote_raw]: + if path: + oc_exec("rm", "-rf", path, check=False, **kwargs) + + return "Cleaned up temp files on Prometheus pod" + + +if __name__ == "__main__": + run.main() diff --git a/projects/cluster/toolbox/create_sa_kubeconfig/main.py b/projects/cluster/toolbox/create_sa_kubeconfig/main.py new file mode 100644 index 000000000..16f1a277e --- /dev/null +++ b/projects/cluster/toolbox/create_sa_kubeconfig/main.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import base64 +import json +import logging +import os +from pathlib import Path + +import yaml + +from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists + +logger = logging.getLogger("DSL") + + +@entrypoint +def run( + sa_name: str, + *, + namespace: str = "", + kubeconfig_file: str, + create_token_secret: bool = True, + token_duration: str = "8760h", +) -> int: + """ + Generate kubeconfig file from a Service Account using non-persisting credential delivery. + + This tool generates a working kubeconfig file that can be used to authenticate as a specific + Service Account. It supports both long-lived token secrets and temporary tokens. + + Args: + sa_name: Name of the Service Account + namespace: Namespace where the Service Account exists/will be created (default: current project) + kubeconfig_file: Output file path for working kubeconfig + create_token_secret: Create a long-lived token secret instead of using kubectl create token + token_duration: Duration for temporary tokens when create_token_secret=False (default: 8760h) + """ + + execute_tasks(locals()) + return 0 + + +@task +def setup_configuration(args, ctx): + """Setup configuration and validate parameters""" + + # Set default namespace from current project if not provided + if args.namespace: + ctx.namespace = args.namespace + else: + result = oc("project", "-q") + ctx.namespace = result.stdout.strip() + if not ctx.namespace: + raise ValueError("Couldn't find the current namespace") + + return f"Prepared to create kubeconfig for SA {args.sa_name} in namespace {ctx.namespace}" + + +@task +def validate_parameters(args, ctx): + """Validate command parameters""" + + if not args.sa_name: + raise ValueError("sa_name cannot be empty") + + # Validate token duration format for temporary tokens + if not args.create_token_secret: + if not args.token_duration.endswith(("s", "m", "h")): + raise ValueError( + "token_duration must end with 's', 'm', or 'h' (e.g., '1h', '30m', '3600s')" + ) + + return f"Validated parameters for Service Account {args.sa_name}" + + +@task +def ensure_service_account(args, ctx): + """Create Service Account if it doesn't exist""" + + if oc_resource_exists("serviceaccount", args.sa_name, namespace=ctx.namespace): + return f"Service Account {args.sa_name} already exists in namespace {ctx.namespace}" + + oc( + "create", + "serviceaccount", + args.sa_name, + "-n", + ctx.namespace, + ) + + return f"Created Service Account {args.sa_name} in namespace {ctx.namespace}" + + +@task +def create_token_secret(args, ctx): + """Create a token secret for the Service Account if requested""" + + if not args.create_token_secret: + return "Using temporary token approach, skipping token secret creation" + + ctx.secret_name = f"{args.sa_name}-token" + + # Check if secret already exists + if oc_resource_exists("secret", ctx.secret_name, namespace=ctx.namespace): + return f"Token secret {ctx.secret_name} already exists in namespace {ctx.namespace}" + + # Create token secret manifest + secret_manifest = { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": ctx.secret_name, + "namespace": ctx.namespace, + "annotations": {"kubernetes.io/service-account.name": args.sa_name}, + }, + "type": "kubernetes.io/service-account-token", + } + + # Serialize manifest to YAML and apply through stdin + secret_yaml = yaml.dump(secret_manifest, default_flow_style=False) + oc("apply", "-f", "-", input_text=secret_yaml, handled_secretly=True) + + return f"Created token secret {ctx.secret_name}" + + +@retry(attempts=30, delay=2, backoff=1.0) +@task +def wait_for_token_secret_ready(args, ctx): + """Wait for token secret to be populated""" + + if not args.create_token_secret: + return "Using temporary token approach, no secret to wait for" + + try: + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + return False, f"Secret {ctx.secret_name} not found" + + data = secret_data.get("data", {}) + if not data.get("token") or not data.get("ca.crt"): + return False, f"Secret {ctx.secret_name} not fully populated yet" + + return f"Token secret {ctx.secret_name} is ready" + + except Exception as e: + return False, f"Error checking secret {ctx.secret_name}: {e}" + + +@task +def extract_cluster_info(args, ctx): + """Extract cluster API server URL""" + + # Get API server URL + result = oc("config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}") + ctx.api_server = result.stdout.strip() + + if not ctx.api_server: + raise RuntimeError("Could not extract API server URL from current kubeconfig") + + return f"Extracted cluster info: API server {ctx.api_server}" + + +def _build_cluster_config(ctx): + """Build cluster configuration section for kubeconfig""" + # Use certificate verification by default for security + config = {"server": ctx.api_server} + + # Check if current KUBECONFIG already uses insecure settings (development context) + is_development_context = _is_development_cluster() + + if is_development_context: + config["insecure-skip-tls-verify"] = True + logger.warning("TLS verification disabled - detected development/testing environment") + + return config + + +def _is_development_cluster() -> bool: + """Determine if we're connecting to a development cluster based on current KUBECONFIG.""" + try: + # Get current cluster configuration + result = oc("config", "view", "--minify", "-o", "json", log_stdout=False) + if result.returncode != 0: + return False + + current_config = json.loads(result.stdout) + clusters = current_config.get("clusters", []) + + if not clusters: + return False + + cluster_config = clusters[0].get("cluster", {}) + + if cluster_config.get("insecure-skip-tls-verify"): + logger.debug( + "Development context detected: current cluster uses insecure-skip-tls-verify" + ) + return True + + return False + + except Exception as e: + logger.debug(f"Could not determine cluster context: {e}") + return False + + +@task +def generate_kubeconfig(args, ctx): + """Generate working kubeconfig file using non-persisting credential delivery""" + + # Extract token locally (not stored in context) + if args.create_token_secret: + # Extract token from secret (with secret handling flag to prevent logging) + secret_data = oc_get_json( + "secret", name=ctx.secret_name, namespace=ctx.namespace, handled_secretly=True + ) + if not secret_data: + raise RuntimeError(f"Token secret {ctx.secret_name} not found") + + token_b64 = secret_data.get("data", {}).get("token") + if not token_b64: + raise RuntimeError(f"Token not found in secret {ctx.secret_name}") + + token = base64.b64decode(token_b64).decode("utf-8") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + else: + # Create temporary token + result = oc( + "create", + "token", + args.sa_name, + "-n", + ctx.namespace, + f"--duration={args.token_duration}", + handled_secretly=True, + ) + + token = result.stdout.strip() + if not token: + raise RuntimeError("Failed to create temporary token") + + # Validate token format and length for security + if len(token) > 10000: # Prevent DoS with excessively long tokens + raise RuntimeError("Token is unexpectedly long - potential security issue") + + # Create working kubeconfig with token + working_kubeconfig = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "cluster": _build_cluster_config(ctx), + "name": "cluster", + } + ], + "contexts": [ + { + "context": { + "cluster": "cluster", + "namespace": ctx.namespace, + "user": args.sa_name, + }, + "name": args.sa_name, + } + ], + "current-context": args.sa_name, + "users": [{"name": args.sa_name, "user": {"token": token}}], # Working token included + } + + # Write working kubeconfig to specified file path with secure permissions + kubeconfig_path = Path(args.kubeconfig_file) + + with open(kubeconfig_path, "w") as f: + yaml.dump(working_kubeconfig, f, default_flow_style=False) + + # Ensure permissions are correctly set after writing + os.chmod(kubeconfig_path, 0o600) + + # Store path for reference (no tokens or structure stored in context) + ctx.kubeconfig_path = str(kubeconfig_path) + + return f"Generated working kubeconfig: {ctx.kubeconfig_path}" + + +@task +def validate_kubeconfig(args, ctx): + """Validate that the generated kubeconfig works""" + + # Test the kubeconfig by running a simple command + + result = oc( + "--kubeconfig", + ctx.kubeconfig_path, + "whoami", + check=True, + ) + + if result.returncode == 0 and result.stdout.strip(): + username = result.stdout.strip() + return f"Kubeconfig validation successful - authenticated as: {username}" + else: + raise RuntimeError("Authentication test returned no username") + + +@task +def display_usage_instructions(args, ctx): + """Display usage instructions""" + + instructions = f""" +# Working kubeconfig created successfully! + +## Usage: +# Use the kubeconfig file directly +export KUBECONFIG={ctx.kubeconfig_path} +oc get pods + +# Or specify the kubeconfig file explicitly +oc --kubeconfig={ctx.kubeconfig_path} get pods + +## Files created: +- {ctx.kubeconfig_path} (working kubeconfig file with authentication) + +## Security Note: +- Kubeconfig file contains authentication token for the Service Account +- File has restrictive permissions (0600) - only readable by owner +- Uses non-persisting credential delivery (token not stored in execution context) +""" + + if args.create_token_secret: + instructions += f"\n## Token source: Long-lived secret {ctx.secret_name}" + else: + instructions += f"\n## Token source: Temporary token (expires in {args.token_duration})" + + print(instructions) + + return f"Working kubeconfig ready: {ctx.kubeconfig_path}" + + +if __name__ == "__main__": + run.main() diff --git a/projects/cluster/toolbox/deploy_custom_catalog/main.py b/projects/cluster/toolbox/deploy_custom_catalog/main.py index eea38593d..113913a54 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/main.py +++ b/projects/cluster/toolbox/deploy_custom_catalog/main.py @@ -17,6 +17,7 @@ def run( catalog_image: str, *, display_name: str = "", + publisher: str = "", ) -> int: """ Deploy a CatalogSource from a custom index image and wait for it to become READY. @@ -26,6 +27,7 @@ def run( catalog_namespace: Namespace where the CatalogSource will be deployed catalog_image: Index image backing the CatalogSource display_name: Optional human-friendly label for logs + publisher: Optional publisher label for the CatalogSource """ execute_tasks(locals()) @@ -75,6 +77,50 @@ def apply_catalog_source(args, ctx): return f"Applied CatalogSource {args.catalog_source_name}" +@retry(attempts=24, delay=5) # 2 minutes total (24 * 5s) +@task +def wait_for_catalog_source_pod_running(args, ctx): + """Wait for the CatalogSource pod to start running""" + + # Search for pod with the catalog source label + result = shell.run( + f"oc get pods -n {args.catalog_namespace} -l olm.catalogSource={args.catalog_source_name} --no-headers", + check=False, + ) + if not result.success: + return False + + # Check if we have actual pods (not "No resources found") + if "No resources found" in result.stdout: + logger.info(f"No pods found yet for catalogSource {args.catalog_source_name}") + return False + + # Count pods and check for multiple matches + pod_count = len(result.stdout.strip().splitlines()) + if pod_count != 1: + logger.warning( + f"Multiple pods ({pod_count}) found for catalogSource {args.catalog_source_name}" + ) + + # Check for image issues + if "ErrImagePull" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull error: {result.stdout.strip()}") + + if "ImagePullBackOff" in result.stdout: + raise RuntimeError(f"CatalogSource pod image pull back off: {result.stdout.strip()}") + + # Check for failed pods first - any failure should cause immediate error + if "Failed" in result.stdout: + raise RuntimeError(f"CatalogSource pod failed to start: {result.stdout.strip()}") + + # Look for Running status in the output + if "Running" in result.stdout: + return "CatalogSource pod is running" + + # Pods exist but not running yet + return False + + @retry(attempts=60, delay=15) @task def wait_for_catalog_source_ready(args, ctx): diff --git a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 index c3244fca4..8f00bdc1a 100644 --- a/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 +++ b/projects/cluster/toolbox/deploy_custom_catalog/templates/catalogsource.yaml.j2 @@ -7,3 +7,6 @@ spec: sourceType: grpc image: {{ args.catalog_image }} displayName: {{ ctx.display_name }} +{% if args.publisher %} + publisher: {{ args.publisher }} +{% endif %} diff --git a/projects/cluster/toolbox/rebuild_image/README.md b/projects/cluster/toolbox/rebuild_image/README.md index b418ba72a..062defce5 100644 --- a/projects/cluster/toolbox/rebuild_image/README.md +++ b/projects/cluster/toolbox/rebuild_image/README.md @@ -43,11 +43,7 @@ Can be imported and used programmatically: ```python from projects.cluster.toolbox.rebuild_image.main import run as rebuild_image -result = rebuild_image( - build_name="my-build", - namespace="my-namespace", - timeout_minutes=30 -) +result = rebuild_image(build_name="my-build", namespace="my-namespace", timeout_minutes=30) ``` ## Common Use Cases @@ -55,4 +51,4 @@ result = rebuild_image( - **FORGE image rebuilds**: Rebuild FORGE container after code changes - **Dependency updates**: Trigger rebuilds when base images change - **Testing**: Validate Build configurations work correctly -- **CI/CD integration**: Automated rebuilds in pipelines \ No newline at end of file +- **CI/CD integration**: Automated rebuilds in pipelines diff --git a/projects/core/CHANGELOG-config.md b/projects/core/CHANGELOG-config.md new file mode 100644 index 000000000..3fbf66f62 --- /dev/null +++ b/projects/core/CHANGELOG-config.md @@ -0,0 +1,17 @@ +# Configuration Framework Changelog + +## 2026-07-16 - Variables Override Helper + +### New Features +- **Variables Override Helper**: Added `write_variables_override()` function for programmatic configuration override generation + - **Behavior**: Creates variables_override.yaml files without requiring full project config initialization + - **Usage**: Supports both preset configuration (`project.args`) and additional variable overrides + - **Error Handling**: Validates `env.ARTIFACT_DIR` initialization with clear error messages + +### Files Modified +- `projects/core/library/config.py` - Added `write_variables_override()` helper function + +### Benefits +- **Early Configuration**: Enables configuration override before project initialization +- **Flexible Override Structure**: Supports both preset lists and arbitrary configuration variables +- **Robust Error Handling**: Fails fast with clear messages when environment not properly initialized diff --git a/projects/core/CHANGELOG-dsl.md b/projects/core/CHANGELOG-dsl.md index b3f69351c..a3c7cd951 100644 --- a/projects/core/CHANGELOG-dsl.md +++ b/projects/core/CHANGELOG-dsl.md @@ -42,6 +42,7 @@ ```python from projects.core.dsl import EarlyReturn + @task def check_operator_deployed(args, ctx): if operator_exists(): diff --git a/projects/core/ci_entrypoint/fournos_resolve.py b/projects/core/ci_entrypoint/fournos_resolve.py index 0d5a39fa1..f864243d8 100644 --- a/projects/core/ci_entrypoint/fournos_resolve.py +++ b/projects/core/ci_entrypoint/fournos_resolve.py @@ -183,11 +183,9 @@ def resolve_fournos_config( # Show the updated spec logger.info("Updated FournosJob spec:") - print("=" * 60) - print("Updated FournosJob spec:") - print("=" * 60) - print(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) - print("=" * 60) + logger.info("=" * 60) + logger.info(yaml.dump({"spec": fjob_obj["spec"]}, default_flow_style=False, sort_keys=False)) + logger.info("=" * 60) if dry_run: logger.info("DRY RUN: Not applying changes to cluster") diff --git a/projects/core/ci_entrypoint/prepare_ci.py b/projects/core/ci_entrypoint/prepare_ci.py index 076d3b01e..11dad7411 100755 --- a/projects/core/ci_entrypoint/prepare_ci.py +++ b/projects/core/ci_entrypoint/prepare_ci.py @@ -783,12 +783,17 @@ def postchecks( """ artifact_dir = os.environ.get("ARTIFACT_DIR") + # Build command string for display + command_str = f"{project} {operation}" + if args: + command_str += f" {' '.join(args)}" + if not artifact_dir: # No artifact dir, just return simple status return ( - f"✅ {project} {operation} completed successfully" + f"✅ {command_str} completed successfully" if finish_reason == FinishReason.SUCCESS - else f"❌ {project} {operation} failed" + else f"❌ {command_str} failed" ) artifact_path = Path(artifact_dir) @@ -821,8 +826,8 @@ def postchecks( if finish_reason != FinishReason.SUCCESS or ( failures_file.exists() and failures_file.stat().st_size > 0 ): - status = f"❌ Execution of '{project} {operation}' failed after{duration_str}." + status = f"❌ Execution of '{command_str}' failed after{duration_str}." else: - status = f"✅ Execution of '{project} {operation}' succeeded after{duration_str}." + status = f"✅ Execution of '{command_str}' succeeded after{duration_str}." return status diff --git a/projects/core/dsl/cli.py b/projects/core/dsl/cli.py index cc1617a47..fc902637b 100644 --- a/projects/core/dsl/cli.py +++ b/projects/core/dsl/cli.py @@ -216,6 +216,7 @@ def resolve_annotation(annotation): cli_name, type=arg_type, required=not has_default, + default=param.default if has_default else None, dest=param_name, help=help_text, ) diff --git a/projects/core/dsl/log.py b/projects/core/dsl/log.py index 0f3c2d18d..16f619975 100644 --- a/projects/core/dsl/log.py +++ b/projects/core/dsl/log.py @@ -104,7 +104,9 @@ def log_execution_banner(function_args: dict = None, log_file: str = None): logger.info("") -def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): +def log_completion_banner( + function_args: dict = None, status: str = "SUCCESS", start_time: float = None +): """Log the completion banner with function info and completion status""" frame = inspect.currentframe() caller_frame = frame.f_back.f_back @@ -118,6 +120,24 @@ def log_completion_banner(function_args: dict = None, status: str = "SUCCESS"): logger.info(f"| {rel_filename}") logger.info(f"| STATUS: {status}") logger.info(f"| COMMAND: {function_name}") + + # Add duration if start_time is provided + if start_time is not None: + end_time = time.time() + duration = end_time - start_time + minutes, seconds = divmod(duration, 60) + hours, minutes = divmod(minutes, 60) + + # Format duration nicely + if hours > 0: + duration_str = f"{int(hours)}h {int(minutes)}m {int(seconds)}s" + elif minutes > 0: + duration_str = f"{int(minutes)}m {int(seconds)}s" + else: + duration_str = f"{duration:.1f}s" + + logger.info(f"| DURATION: {duration_str}") + logger.info(f"| ARTIFACTS: {env.ARTIFACT_DIR}") logger.info("===============================================================================") logger.info("") diff --git a/projects/core/dsl/runtime.py b/projects/core/dsl/runtime.py index 024555f64..c0c4b8b88 100644 --- a/projects/core/dsl/runtime.py +++ b/projects/core/dsl/runtime.py @@ -174,7 +174,7 @@ def execute_tasks(function_args: dict = None): if not file_tasks: logger.error(f"No tasks found for file: {rel_filename}") - log_completion_banner(function_args, status="NO_TASKS") + log_completion_banner(function_args, status="NO_TASKS", start_time=start_time) raise RuntimeError(f"No tasks found for file: {rel_filename}") execution_error = None @@ -193,7 +193,7 @@ def execute_tasks(function_args: dict = None): logger.fatal("==> INTERRUPTED: Received KeyboardInterrupt (Ctrl+C)") logger.info("==> Exiting...") # Show completion banner with interrupted status - log_completion_banner(function_args, status="INTERRUPTED") + log_completion_banner(function_args, status="INTERRUPTED", start_time=start_time) raise except EarlyReturnException as e: @@ -213,7 +213,11 @@ def execute_tasks(function_args: dict = None): # Log error but continue to always tasks logger.info("") logger.fatal(f"==> {e.__class__.__name__}: {e}") - log_completion_banner(function_args, status=f"EXCEPTION ({e.__class__.__name__})") + log_completion_banner( + function_args, + status=f"EXCEPTION ({e.__class__.__name__})", + start_time=start_time, + ) # Save error to re-raise after always tasks execute execution_error = e @@ -278,6 +282,7 @@ def execute_tasks(function_args: dict = None): log_completion_banner( function_args, status=f"FAILED ({len(all_exceptions)} exception(s))", + start_time=start_time, ) if len(all_exceptions) == 1: raise all_exceptions[0] @@ -287,7 +292,7 @@ def execute_tasks(function_args: dict = None): ) # Log completion banner if execution was successful - log_completion_banner(function_args) + log_completion_banner(function_args, start_time=start_time) # for the time being, return the shared context. In the # future we'll return more info about what has been @@ -669,7 +674,7 @@ def _generate_failure_file_for_agent(exception: Exception) -> None: with open(failure_file_path, "w", encoding="utf-8") as f: f.write(failure_content) - logger.info(f"Generated FAILURE file for agent analysis: {failure_file_path}") + logger.debug(f"Generated FAILURE file for agent analysis: {failure_file_path}") except Exception as failure_error: logger.warning(f"Failed to generate FAILURE file for agent: {failure_error}") diff --git a/projects/core/dsl/shell.py b/projects/core/dsl/shell.py index fbd833587..019a0feec 100644 --- a/projects/core/dsl/shell.py +++ b/projects/core/dsl/shell.py @@ -32,6 +32,9 @@ def run( stdout_dest: str | Path | None = None, log_stdout: bool = True, log_stderr: bool = True, + input_text: str | None = None, + timeout_seconds: float | None = None, + text: bool = True, ) -> CommandResult: """ Execute a shell command @@ -43,6 +46,9 @@ def run( stdout_dest: Optional file path to write stdout to log_stdout: Optional. If False, don't log the content of stdout. log_stderr: Optional. If False, don't log the content of stderr. + input_text: Optional text to send to command's stdin + timeout_seconds: Optional timeout in seconds + text: Optional. If False, handle binary output (default True) Returns: CommandResult with execution details """ @@ -64,12 +70,23 @@ def run( shell=shell, check=False, # We handle check ourselves capture_output=True, - text=True, + input=input_text, + timeout=timeout_seconds, + text=text, ) + # Handle text vs binary output + if text: + stdout_str = result.stdout or "" + stderr_str = result.stderr or "" + else: + # For binary mode, convert to string representation for CommandResult + stdout_str = f"" if result.stdout else "" + stderr_str = result.stderr.decode("utf-8", errors="replace") if result.stderr else "" + cmd_result = CommandResult( - stdout=result.stdout or "", - stderr=result.stderr or "", + stdout=stdout_str, + stderr=stderr_str, returncode=result.returncode, command=command, ) @@ -78,34 +95,60 @@ def run( if stdout_dest: stdout_path = Path(stdout_dest) stdout_path.parent.mkdir(parents=True, exist_ok=True) - with open(stdout_path, "w") as f: - f.write(cmd_result.stdout) + with open(stdout_path, "w" if text else "wb") as f: + f.write(stdout_str if text else result.stdout) # Print output in verbose format - if result.stdout and result.stdout.strip(): - if stdout_dest: - logger.info(f"| ") - elif log_stdout: - stdout = result.stdout.strip().splitlines() - if len(stdout) == 1: - logger.info(f"| {stdout[0]}") + if text: + if result.stdout: + if stdout_dest: + logger.info(f"| ") + elif log_stdout: + stdout_lines = stdout_str.strip().splitlines() + if len(stdout_lines) == 1: + logger.info(f"| {stdout_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stdout_lines) + "\n| ") + else: + logger.info("| ") + + if result.stderr: + if log_stderr: + stderr_lines = stderr_str.strip().splitlines() + if len(stderr_lines) == 1: + logger.info(f"| {stderr_lines[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr_lines) + "\n| ") else: - logger.info("| \n" + "\n| ".join(stdout) + "\n| ") - else: - logger.info("| ") - - if result.stderr and result.stderr.strip(): - if log_stderr: - stderr = result.stderr.strip().splitlines() - if len(stderr) == 1: - logger.info(f"| {stderr[0]}") + logger.info("| ") + + if not (result.stdout or result.stderr): + logger.info("| ") + else: + # Binary mode - always show lengths + if result.stdout: + if stdout_dest: + logger.info( + f"| ({len(result.stdout)} bytes)" + ) else: - logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + logger.info(f"| {len(result.stdout)} bytes") - if not (result.stdout.strip() or result.stderr.strip()): - logger.info("| ") + if result.stderr: + # Try to decode stderr as text for logging (it's usually text even in binary mode) + try: + stderr_text = result.stderr.decode("utf-8", errors="replace").strip() + if stderr_text: + stderr = stderr_text.splitlines() + if len(stderr) == 1: + logger.info(f"| {stderr[0]}") + else: + logger.info("| \n" + "\n| ".join(stderr) + "\n| ") + except Exception: + logger.info(f"| {len(result.stderr)} bytes") + + if not (result.stdout or result.stderr): + logger.info("| ") if result.returncode != 0: logger.info(f"| {result.returncode}") @@ -116,9 +159,9 @@ def run( # Create a more informative error message error_msg = f"Command failed with exit code {result.returncode}: {command}" if result.stderr: - error_msg += f"\nSTDERR: {result.stderr.strip()}" + error_msg += f"\nSTDERR: {stderr_str.strip()}" if result.stdout: - error_msg += f"\nSTDOUT: {result.stdout.strip()}" + error_msg += f"\nSTDOUT: {stdout_str.strip()}" # Create exception with enhanced message error = subprocess.CalledProcessError( diff --git a/projects/core/dsl/task.py b/projects/core/dsl/task.py index 40e31017c..bd593a512 100644 --- a/projects/core/dsl/task.py +++ b/projects/core/dsl/task.py @@ -59,6 +59,7 @@ def _log_retry_attempt( result=None, exc=None, artifact_dirname_suffix=None, + retry_backoff=1.0, ): """ Log a retry attempt with consistent formatting. @@ -115,7 +116,21 @@ def _log_retry_attempt( retry_reason = f"returned: {result}" logger.warning(f"~~ RETRY ATTEMPT #{attempt + 1}/{retry_attempts} ({retry_reason})") - logger.info(f"~~ RETRY in {current_delay:.0f}s") + + # Calculate total remaining time for all future retries with backoff + remaining_attempts = retry_attempts - 1 - attempt + total_remaining_time = 0 + delay_for_calculation = current_delay + + for _i in range(remaining_attempts): + total_remaining_time += delay_for_calculation + delay_for_calculation *= retry_backoff + + # Calculate when retries will end + end_time = time.time() + total_remaining_time + end_timestamp = datetime.fromtimestamp(end_time).strftime("%H:%M") + + logger.info(f"~~ RETRY in {current_delay:.0f}s until {end_timestamp}") logger.info("~" * LINE_WIDTH) logger.info("") @@ -187,6 +202,7 @@ def _execute_with_retry( retry_reason=retry_reason, result=result, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff else: @@ -239,6 +255,7 @@ def _execute_with_retry( current_delay, exc=exc, artifact_dirname_suffix=artifact_dirname_suffix, + retry_backoff=retry_backoff, ) current_delay *= retry_backoff diff --git a/projects/core/dsl/utils/__init__.py b/projects/core/dsl/utils/__init__.py index 296b0cccd..80effeb96 100644 --- a/projects/core/dsl/utils/__init__.py +++ b/projects/core/dsl/utils/__init__.py @@ -39,5 +39,13 @@ def slugify_identifier(value: str, *, max_length: int = 63) -> str: def truncate_k8s_name(value: str, *, max_length: int = 63) -> str: - """Truncate a string to Kubernetes name length limits.""" - return value[:max_length].rstrip("-") + """Truncate a string to Kubernetes name length limits. + + When truncation occurs, a short hash suffix is appended to avoid + collisions between names that share a long common prefix. + """ + if len(value) <= max_length: + return value + hash_suffix = hashlib.sha256(value.encode()).hexdigest()[:6] + prefix = value[: max_length - len(hash_suffix) - 1].rstrip("-") + return f"{prefix}-{hash_suffix}" diff --git a/projects/core/dsl/utils/k8s.py b/projects/core/dsl/utils/k8s.py index 460ff0755..a843da00c 100644 --- a/projects/core/dsl/utils/k8s.py +++ b/projects/core/dsl/utils/k8s.py @@ -23,29 +23,56 @@ def oc( timeout_seconds: float | None = 300, log_stdout: bool = True, log_stderr: bool = True, + stdout_dest: Path | None = None, + text: bool = True, + handled_secretly: bool = False, ) -> shell.CommandResult: """Run an oc command. Args: *args: Arguments to pass to oc check: Raise subprocess.CalledProcessError if command fails - input_text: Input to send to command (not supported) - timeout_seconds: Command timeout (not supported) + input_text: Input to send to command + timeout_seconds: Command timeout log_stdout: Log stdout to console (default True) log_stderr: Log stderr to console (default True) + stdout_dest: Optional file path to write stdout to + text: Optional. If False, handle binary output (default True) + handled_secretly: If True, disable all logging to prevent secret leakage Returns: CommandResult with execution details """ cmd = ["oc", *args] - return shell.run( - cmd, - check=check, - shell=False, - log_stdout=log_stdout, - log_stderr=log_stderr, - ) + # When handled secretly, disable all logging to prevent credential leakage + if handled_secretly: + log_stdout = False + log_stderr = False + + try: + return shell.run( + cmd, + check=check, + shell=False, + log_stdout=log_stdout, + log_stderr=log_stderr, + stdout_dest=stdout_dest, + input_text=input_text, + timeout_seconds=timeout_seconds, + text=text, + ) + except subprocess.CalledProcessError as e: + if not handled_secretly: + raise + + # When handled secretly, sanitize error to prevent credential leakage + raise subprocess.CalledProcessError( + e.returncode, + "oc command failed", + "", + "Error output suppressed for secret handling", + ) from e def oc_get_json( @@ -55,6 +82,7 @@ def oc_get_json( namespace: str | None = None, selector: str | None = None, ignore_not_found: bool = False, + handled_secretly: bool = False, ) -> dict[str, Any] | None: """Get a Kubernetes resource as JSON using oc. @@ -64,6 +92,7 @@ def oc_get_json( namespace: Namespace (optional) selector: Label selector (optional) ignore_not_found: Return None instead of raising error if not found + handled_secretly: If True, disable all logging to prevent secret leakage Returns: Resource data as dict, or None if not found and ignore_not_found=True @@ -80,20 +109,30 @@ def oc_get_json( args.extend(["-l", selector]) args.extend(["-o", "json"]) + # When handled secretly, disable all logging to prevent credential leakage + log_stderr_enabled = not ignore_not_found and not handled_secretly + result = oc( *args, check=not ignore_not_found, - log_stdout=False, - log_stderr=not ignore_not_found, + log_stdout=False, # Always disabled to prevent data exposure + log_stderr=log_stderr_enabled, # Disabled when handled secretly + handled_secretly=handled_secretly, ) if result.returncode != 0: if ignore_not_found and _is_oc_not_found_error(result.stderr): return None + # Re-raise error (oc() already sanitized if handled_secretly=True) raise subprocess.CalledProcessError( result.returncode, f"oc {' '.join(args)}", result.stdout, result.stderr ) if not result.stdout: - raise ValueError(f"oc {' '.join(args)} returned no output") + error_msg = ( + f"oc get {kind} returned no output" + if handled_secretly + else f"oc {' '.join(args)} returned no output" + ) + raise ValueError(error_msg) return json.loads(result.stdout) @@ -266,6 +305,7 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: logger.info("Capturing logs from %d pods in %s", len(pod_names), namespace) for pod_name in pod_names: + log_file = output_dir / f"{pod_name}.log" log_result = oc( "logs", pod_name, @@ -274,11 +314,10 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: "--all-containers=true", check=False, log_stdout=False, + stdout_dest=log_file, ) - log_file = output_dir / f"{pod_name}.log" - if log_result.returncode == 0 and log_result.stdout: - log_file.write_text(log_result.stdout, encoding="utf-8") - else: + # If command failed and no output was written, write error info + if log_result.returncode != 0 and (not log_file.exists() or log_file.stat().st_size == 0): log_file.write_text( f"(failed to collect logs: exit_code={log_result.returncode})\n" f"{log_result.stderr or ''}", @@ -286,6 +325,59 @@ def capture_pod_logs(*, namespace: str, output_dir: Path) -> None: ) +def oc_exec( + *command: str, + namespace: str, + pod: str, + container: str | None = None, + **kwargs, +) -> shell.CommandResult: + """Exec a command inside a pod container. + + Args: + *command: Command and arguments to run in the pod + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + **kwargs: Additional keyword arguments passed to oc() + + Returns: + CommandResult with execution details + """ + args = ["-n", namespace, "exec", pod] + if container: + args.extend(["-c", container]) + args.append("--") + args.extend(command) + return oc(*args, **kwargs) + + +def oc_cp_from_pod( + remote_path: str, + local_path: str | Path, + *, + namespace: str, + pod: str, + container: str | None = None, +) -> shell.CommandResult: + """Copy a file from a pod to a local path. + + Args: + remote_path: Absolute path inside the pod + local_path: Local destination path + namespace: Namespace of the pod + pod: Pod name + container: Container name (optional if pod has a single container) + + Returns: + CommandResult with execution details + """ + args = ["cp", f"{namespace}/{pod}:{remote_path}", str(local_path)] + if container: + args.extend(["-c", container]) + return oc(*args) + + def best_effort_oc(*oc_args: str, description: str | None = None) -> None: """Run an oc command, swallowing timeout and other errors. diff --git a/projects/core/library/ci.py b/projects/core/library/ci.py index 475da31eb..894e5c7d4 100644 --- a/projects/core/library/ci.py +++ b/projects/core/library/ci.py @@ -108,11 +108,16 @@ def _display_error_summary(e: Exception) -> None: for line in summary_lines: logger.error(line) - if env.ARTIFACT_DIR is None: - logging.error("env.ARTIFACT_DIR not set, cannot generate the error summary file") - else: - # Write to FAILURES file - _write_error_summary_to_file(summary_lines) + # Attempt to write to FAILURES file if environment is initialized + try: + if env.ARTIFACT_DIR is not None: + _write_error_summary_to_file(summary_lines) + else: + logger.warning( + "Environment not fully initialized - error summary displayed above but not saved to file" + ) + except Exception as write_error: + logger.warning(f"Could not save error summary to file: {write_error}") logger.error("=" * 80) diff --git a/projects/core/library/config.py b/projects/core/library/config.py index d06fca5e2..0d1fa1303 100644 --- a/projects/core/library/config.py +++ b/projects/core/library/config.py @@ -210,7 +210,7 @@ def apply_preset(self, name): for key, value in values.items(): if key == "extends": - for extend_name in value: + for extend_name in value or []: self.apply_preset(extend_name) continue @@ -519,3 +519,60 @@ def reload(orchestration_dir, *, apply_config_overrides=True): init(orchestration_dir, apply_config_overrides=apply_config_overrides) return project + + +def write_variables_override(presets=None, variables_dict=None): + """Write configuration overrides to the variables_override.yaml file in ARTIFACT_DIR. + + This is a top-level function that doesn't require project config to be initialized. + + Args: + presets: List of presets to set in project.args + variables_dict: Dictionary of additional configuration paths and values + """ + from pathlib import Path + + if not env.ARTIFACT_DIR: + raise ValueError( + "env.ARTIFACT_DIR is not initialized. " + "Call env.init() before using write_variables_override()" + ) + + artifact_dir = Path(env.ARTIFACT_DIR) + override_file = artifact_dir / VARIABLE_OVERRIDES_FILENAME + + # Ensure the directory exists + override_file.parent.mkdir(parents=True, exist_ok=True) + + # Load existing overrides or create empty structure + if override_file.exists(): + with open(override_file) as f: + loaded = yaml.safe_load(f) + if loaded is not None and not isinstance(loaded, dict): + raise ValueError( + f"Override file {override_file} must contain a YAML dictionary, got {type(loaded).__name__}" + ) + overrides = loaded or {} + else: + overrides = {} + + # Set presets in project.args + if presets: + if len(presets) == 1 and "," in presets[0]: + presets = list(presets[0].split(",")) + + overrides["project.args"] = presets + logger.info(f"write_variables_override: project.args --> {presets}") + + # Add additional variables to top level + if variables_dict: + for jsonpath, value in variables_dict.items(): + overrides[jsonpath] = value + logger.info(f"write_variables_override: {jsonpath} --> {value}") + + # Write back to file + with open(override_file, "w") as f: + yaml.dump(overrides, f, indent=4, default_flow_style=False, sort_keys=False) + + total_items = (1 if presets else 0) + (len(variables_dict) if variables_dict else 0) + logger.info(f"write_variables_override: wrote {total_items} override(s) to {override_file}") diff --git a/projects/core/library/env.py b/projects/core/library/env.py index 7c07ca732..e2ef4bbb2 100644 --- a/projects/core/library/env.py +++ b/projects/core/library/env.py @@ -3,6 +3,8 @@ import pathlib import threading import time +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO # ARTIFACT_DIR is accessed via __getattr__ for thread-local support BASE_ARTIFACT_DIR = None # Immutable copy of the initial ARTIFACT_DIR @@ -33,7 +35,7 @@ def __getattr__(name): ) return globals().get("_GLOBAL_ARTIFACT_DIR") - return globals()[name] + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") def get_tls_artifact_dir(): @@ -98,11 +100,24 @@ def init(daily_artifact_dir=False): else: env_forge_base_dir = pathlib.Path(os.environ.get("FORGE_BASE_DIR", "/tmp")) - artifact_dir = env_forge_base_dir / f"forge_{time.strftime('%Y%m%d-%H%M')}" + # Try base format first, add seconds if directory exists + base_name = f"forge_{time.strftime('%Y%m%d-%H%M')}" + artifact_dir = env_forge_base_dir / base_name + + if artifact_dir.exists(): + # Directory exists, add seconds to make it unique + unique_name = f"forge_{time.strftime('%Y%m%d-%H%M-%S')}" + artifact_dir = env_forge_base_dir / unique_name artifact_dir.mkdir(parents=True, exist_ok=True) os.environ["ARTIFACT_DIR"] = str(artifact_dir) + # Create or update forge_last symlink + forge_last_link = env_forge_base_dir / "forge_last" + if forge_last_link.exists() or forge_last_link.is_symlink(): + forge_last_link.unlink() + forge_last_link.symlink_to(artifact_dir.name) + artifact_dir.mkdir(parents=True, exist_ok=True) # Set BASE_ARTIFACT_DIR to the initial value (immutable) @@ -202,3 +217,77 @@ def next_artifact_index(): return 0 return len(list(current_artifact_dir.glob("*__*"))) + + +class MuteStdOut: + """Context manager to mute stdout, stderr, and logging with a startup message.""" + + def __init__(self, reason: str): + self.reason = reason + self.captured_stdout = StringIO() + self.captured_stderr = StringIO() + self.original_logging_level = None + self.original_handlers_streams = [] + + def __enter__(self): + # Print the startup message before muting + logger = logging.getLogger(__name__) + logger.info(self.reason) + + # Redirect logging handlers that have direct stream references + self._redirect_logging_handlers() + + # Suppress logging by setting root logger level to CRITICAL + root_logger = logging.getLogger() + self.original_logging_level = root_logger.level + root_logger.setLevel(logging.CRITICAL) + + # Start capturing stdout and stderr + self.stdout_redirect = redirect_stdout(self.captured_stdout) + self.stderr_redirect = redirect_stderr(self.captured_stderr) + + self.stdout_redirect.__enter__() + self.stderr_redirect.__enter__() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Restore stdout and stderr + self.stdout_redirect.__exit__(exc_type, exc_val, exc_tb) + self.stderr_redirect.__exit__(exc_type, exc_val, exc_tb) + + # Restore original logging level + if self.original_logging_level is not None: + root_logger = logging.getLogger() + root_logger.setLevel(self.original_logging_level) + + # Restore original logging handler streams + self._restore_logging_handlers() + + return False + + def _redirect_logging_handlers(self): + """Redirect logging handlers with direct stream references to captured streams.""" + import sys + + # Save and redirect handlers from root logger and all existing loggers + all_loggers = [logging.getLogger()] + [ + logging.getLogger(name) for name in logging.Logger.manager.loggerDict + ] + + for logger in all_loggers: + for handler in getattr(logger, "handlers", []): + if hasattr(handler, "stream"): + original_stream = handler.stream + self.original_handlers_streams.append((handler, original_stream)) + + if original_stream is sys.stdout: + handler.stream = self.captured_stdout + elif original_stream is sys.stderr: + handler.stream = self.captured_stderr + + def _restore_logging_handlers(self): + """Restore original streams to logging handlers.""" + for handler, original_stream in self.original_handlers_streams: + handler.stream = original_stream + self.original_handlers_streams.clear() diff --git a/projects/core/library/export.py b/projects/core/library/export.py index c1eff68f1..a115a84c3 100644 --- a/projects/core/library/export.py +++ b/projects/core/library/export.py @@ -729,6 +729,12 @@ def _build_enhanced_notification( notification_parts.append("**Test Logs**") notification_parts.extend(step_log_links) + # Add distinct test and post-processing status right under Test Logs + test_status_section = _build_test_status_section(status) + if test_status_section: + notification_parts.append("") + notification_parts.extend(test_status_section) + if postprocess_status_links: notification_parts.append("") notification_parts.extend(postprocess_status_links) @@ -740,6 +746,42 @@ def _build_enhanced_notification( return "\n".join(notification_parts) +def _build_test_status_section(status: dict[str, Any]) -> list[str]: + """Build distinct test and post-processing status section.""" + try: + test_phase = status.get("test_phase", {}) + if not test_phase: + return [] + + test_status = test_phase.get("phase", "UNKNOWN") + test_message = test_phase.get("message", "") + + # Determine post-processing status based on final status and test outcome + final_status = status.get("final_status", "unknown") + if test_status == "FAILED": + post_processing_status = "skipped" # Don't run post-processing if test failed + elif final_status == "success": + post_processing_status = "success" + elif "failed" in final_status.lower(): + post_processing_status = "failed" + else: + post_processing_status = "unknown" + + status_lines = [f"**test:** {test_status}"] + + if test_message: + # Format message with blockquote-style prefix + status_lines.append(f"> {test_message}") + + status_lines.append(f"**post-processing:** {post_processing_status}") + + return status_lines + + except Exception as e: + logger.warning(f"Failed to build test status section: {e}") + return [] + + def _extract_project_from_status(status: dict[str, Any]) -> str: """Extract project name from status object or environment.""" # Try to get project from environment variables diff --git a/projects/core/library/postprocess.py b/projects/core/library/postprocess.py index c6ab3186f..8eccb5af4 100644 --- a/projects/core/library/postprocess.py +++ b/projects/core/library/postprocess.py @@ -33,7 +33,12 @@ def write_test_labels( - directory: Path, labels: dict[str, str], *, version: str = "1", dump_config: bool = True + directory: Path, + labels: dict[str, str], + *, + version: str = "1", + dump_config: bool = True, + kpi_labels: dict[str, str] | None = None, ) -> Path: """Write a __test_labels__.yaml file to mark a directory as a Caliper test base. @@ -42,6 +47,7 @@ def write_test_labels( labels: Dictionary of label key-value pairs version: Version string for the test labels format (default: "1") dump_config: Whether to save project configuration to config.yaml (default: True) + kpi_labels: Optional dictionary of KPI labels for system context Returns: Path to the created __test_labels__.yaml file @@ -53,6 +59,10 @@ def write_test_labels( "model": "llama-3", "deployment": "single-zone", "rate": "10" + }, + kpi_labels={ + "platform": "CKS", + "gpu_type": "H100" } ) """ @@ -62,6 +72,9 @@ def write_test_labels( "labels": labels, } + if kpi_labels: + payload["kpi_labels"] = kpi_labels + # Create directory and write YAML test_labels_path.parent.mkdir(parents=True, exist_ok=True) with test_labels_path.open("w", encoding="utf-8") as handle: @@ -143,17 +156,29 @@ def run_and_postprocess(test_func, *args, **kwargs): exc_msg: str | None = None ret: int | None = None original_exc: BaseException | None = None # Store the test exception + interrupted: bool = False try: ret = test_func(*args, **kwargs) return ret - except BaseException as e: + except (KeyboardInterrupt, BaseException) as e: + # Import here to avoid circular dependencies + from projects.core.library.run import SignalInterrupt + + # Check if this is an interrupt + if isinstance(e, KeyboardInterrupt | SignalInterrupt): + interrupted = True + logger.info("==> Test interrupted - skipping Caliper post-processing") + exc_msg = str(e) original_exc = e # Capture before the name is cleared raise finally: - # Determine test outcome based on exception/return code - if exc_msg is not None: + # Skip post-processing when interrupted to avoid blocking on finalizers + if interrupted: + logger.info("==> Skipping Caliper post-processing due to interrupt") + outcome = None + elif exc_msg is not None: outcome = TestPhaseOutcome("FAILED", exc_msg) elif ret == 0: outcome = TestPhaseOutcome("SUCCESS") @@ -164,56 +189,80 @@ def run_and_postprocess(test_func, *args, **kwargs): # Run postprocessing and check status for failures try: - status = run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) - - # Check if postprocessing failed - success = status.get("success", False) - if not success: - final_status = status.get("final_status", "unknown") - - # Check if failure is only due to warnings - if _is_warnings_only_failure(status): - if original_exc is not None: - # Test failed, postprocess has warnings: still fail due to test - logger.error( - "Test failed and postprocessing completed with warnings (final_status: %s)", - final_status, - ) - raise # Re-raise the original test exception - else: - # Test succeeded, postprocess has warnings only: treat as success - logger.warning( - "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", - final_status, - ) - return 0 + status = ( + run_postprocess_after_test(artifact_base_dir, test_outcome=outcome) + if not interrupted + else dict(success=False, final_status="Test interrupted") + ) + + # Handle None status when postprocessing is disabled - treat as unsuccessful + if status is None or not status.get("success", False): + if status is None: + final_status = "postprocessing disabled" + else: + final_status = status.get("final_status", "unknown") + result = _handle_postprocess_failure(status, original_exc, final_status) + if result is None: + # Let original test exception propagate through outer flow + pass else: - # Actual postprocessing failures (not just warnings) - if original_exc is not None: - # Both test and postprocess failed: log both issues - logger.error( - "Both test and postprocessing failed (final_status: %s)", final_status - ) - raise # Re-raise the original test exception - else: - # Only postprocess failed: return failure code - logger.error( - "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", - final_status, - ) - return 1 + return result except Exception as postprocess_exc: logger.exception("Caliper postprocess after test failed with exception") if original_exc is not None: # Both test and postprocess failed: chain so both are visible in the traceback raise postprocess_exc from original_exc - else: - # Only postprocess failed: return failure code instead of raising - logger.error( - "Test succeeded but postprocessing failed with exception - returning exit code 1" - ) - return 1 + + # Only postprocess failed: return failure code instead of raising + logger.error( + "Test succeeded but postprocessing failed with exception - returning exit code 1" + ) + return 1 + + +def _handle_postprocess_failure( + status: dict, original_exc: BaseException | None, final_status: str +) -> int | None: + """Handle postprocessing failure logic. + + Args: + status: Postprocessing status dictionary + original_exc: Original test exception if any + final_status: Final status from postprocessing + + Returns: + Exit code to return, or None to re-raise original exception + """ + # Check if failure is only due to warnings + if _is_warnings_only_failure(status): + if original_exc is not None: + # Test failed, postprocess has warnings: still fail due to test + logger.error( + "Test failed and postprocessing completed with warnings (final_status: %s)", + final_status, + ) + return None # Signal to re-raise original exception + else: + # Test succeeded, postprocess has warnings only: treat as success + logger.warning( + "Test succeeded and postprocessing completed with warnings (final_status: %s) - returning exit code 0", + final_status, + ) + return 0 + else: + # Actual postprocessing failures (not just warnings) + if original_exc is not None: + # Both test and postprocess failed: log both issues + logger.error("Both test and postprocessing failed (final_status: %s)", final_status) + return None # Signal to re-raise original exception + else: + # Only postprocess failed: return failure code + logger.error( + "Test succeeded but postprocessing failed (final_status: %s) - returning exit code 1", + final_status, + ) + return 1 def _is_warnings_only_failure(status: dict) -> bool: @@ -273,21 +322,18 @@ def run_postprocess_after_test( if not postprocess_config.enabled: logger.info("Caliper post-processing disabled (caliper.postprocess.enabled: false).") - return + return dict(success=True, final_status="Post-processing disabled") artifact_root_path = Path(artifact_root).resolve() if artifact_root is not None else None with env.NextArtifactDir("postprocessing"): - workspace = Path(env.ARTIFACT_DIR).resolve() logger.info( - "Running Caliper postprocess (artifacts=%s, workspace=%s, test_phase=%s)", + "Running Caliper postprocess (artifacts=%s, test_phase=%s)", artifact_root_path, - workspace, test_outcome.phase if test_outcome else "SUCCESS", ) status = run_orchestration_postprocess( artifact_dir=artifact_root_path, - visualize_output_dir=workspace, test_outcome=test_outcome, ) logger.info( @@ -415,11 +461,13 @@ def run_orchestration_postprocess( def postprocess_command(_ctx, artifact_dir: Path, output_dir: Path): """Run the post-processing pipeline.""" - status = run_orchestration_postprocess( - artifact_dir=artifact_dir, - test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), - visualize_output_dir=output_dir, - ) + # Set ARTIFACT_DIR temporarily so config resolution works with user's output directory + with env.TempArtifactDir(output_dir): + status = run_orchestration_postprocess( + artifact_dir=artifact_dir, + test_outcome=TestPhaseOutcome("NOT_AVAILABLE"), + visualize_output_dir=None, # Let config resolution handle it + ) logger.info("Caliper postprocess status:\n" + yaml.dump(status, indent=2)) # Check success flag and return appropriate exit code diff --git a/projects/core/library/replot.py b/projects/core/library/replot.py index b2e3cd7fb..81696785b 100644 --- a/projects/core/library/replot.py +++ b/projects/core/library/replot.py @@ -167,9 +167,12 @@ def caliper_replot_entrypoint( logger.info("Replot status:") logger.info(yaml.dump(status, indent=2)) - # Show where artifacts have been saved - final_artifact_dir = artifact_directory or env.ARTIFACT_DIR - logger.info(f"📁 Artifacts saved to: {final_artifact_dir}") + try: + logger.info( + f"📁 Artifacts saved to: {status['replot']['postprocess_result']['base_directory']}" + ) + except Exception: + pass # Check if replot was successful replot_status = status.get("replot", {}).get("status", "unknown") diff --git a/projects/core/nightly/__init__.py b/projects/core/nightly/__init__.py new file mode 100644 index 000000000..c0cdba66f --- /dev/null +++ b/projects/core/nightly/__init__.py @@ -0,0 +1,20 @@ +"""Shared nightly pipeline handlers. + +Architecture: + Core (this package) defines abstract interfaces and the orchestration logic. + Each project implements concrete receivers/verifiers in its nightly_resolvers/ package + and exposes them via get_receiver() and get_verifier() factory functions. + + The core handler imports the project's nightly_resolvers module at runtime and calls + those factories — pure polymorphism, no knowledge of concrete implementations. + +Interfaces: + - ImageReceiver: fetches the latest version from a registry + - NightlyVerifier: checks if a version was already tested + +Handler: + - handler.run(): single phase that resolves version, compares, and creates FournosJob + +Projects register this as a phase in ci_base: + "nightly" -> projects.core.nightly.handler:run +""" diff --git a/projects/core/nightly/base_receiver.py b/projects/core/nightly/base_receiver.py new file mode 100644 index 000000000..2e6e70ab7 --- /dev/null +++ b/projects/core/nightly/base_receiver.py @@ -0,0 +1,21 @@ +"""Abstract interface for image receivers. + +Projects implement this to define how the latest available version +is fetched from a container registry (GHCR, Quay, etc.). +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + + +class ImageReceiver(ABC): + """Fetches the latest available version from a container registry.""" + + @abstractmethod + def get_latest_version(self) -> str: + """Query the registry and return the latest version string. + + Raises on failure (network, auth, no tags found, etc.). + """ + ... diff --git a/projects/core/nightly/base_verifier.py b/projects/core/nightly/base_verifier.py new file mode 100644 index 000000000..de42c2037 --- /dev/null +++ b/projects/core/nightly/base_verifier.py @@ -0,0 +1,22 @@ +"""Abstract interface for nightly verifiers. + +Projects implement this to define how run history is checked +(S3, MLflow, etc.) to determine if a version was already tested. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + + +class NightlyVerifier(ABC): + """Checks historical run data to determine if a version was already tested.""" + + @abstractmethod + def get_last_tested_version(self) -> str: + """Return the version string of the last successful run. + + Raises an exception if the version cannot be determined + (missing credentials, experiment not found, etc.). + """ + ... diff --git a/projects/core/nightly/handler.py b/projects/core/nightly/handler.py new file mode 100644 index 000000000..e8e614659 --- /dev/null +++ b/projects/core/nightly/handler.py @@ -0,0 +1,166 @@ +"""Nightly pipeline handler — single phase that orchestrates: + +1. Resolve the latest available image version (via project's ImageReceiver) +2. Compare against last tested version (via project's NightlyVerifier) +3. Create a FournosJob if a new version is detected + +Error visibility is provided via notification files in the CI metadata directory. +""" + +from __future__ import annotations + +import importlib +import logging +import os + +import yaml + +from projects.core.library import ci as ci_lib +from projects.core.library import config, env +from projects.core.library.config import requires + +logger = logging.getLogger(__name__) + + +def _get_source_config() -> dict: + """Read the source-specific config from nightly.sources..""" + source = config.project.get_config("nightly.source") + return config.project.get_config(f"nightly.sources.{source}") + + +def _resolve_class(dotted_path: str): + """Import a class from a dotted path like 'projects.foo.bar.MyClass'.""" + module_path, class_name = dotted_path.rsplit(".", 1) + mod = importlib.import_module(module_path) + return getattr(mod, class_name) + + +def _receive_image(source_cfg: dict) -> str: + """Resolve the latest available version from the source's receiver.""" + receiver_cls = _resolve_class(source_cfg["receiver"]) + receiver = receiver_cls() + + logger.info("Using receiver: %s", type(receiver).__name__) + version = receiver.get_latest_version() + logger.info("Resolved version: %s", version) + + version_file = env.BASE_ARTIFACT_DIR / "resolved_version.txt" + version_file.write_text(version) + + (env.ARTIFACT_DIR / "resolved_version.txt").write_text(version) + return version + + +def _confirm_nightly(project: str, image_version: str, source_cfg: dict) -> None: + """Compare resolved version against last tested, create FournosJob if new.""" + verifier_cls = _resolve_class(config.project.get_config("nightly.verifier")) + verifier = verifier_cls() + + logger.info("Using verifier: %s", type(verifier).__name__) + try: + last_version = verifier.get_last_tested_version() + except Exception as e: + logger.warning( + "Verifier failed (bootstrap scenario?): %s. " + "Treating as no previous version — will launch a new run.", + e, + ) + last_version = "" + logger.info("Last tested version: '%s'", last_version or "") + + if image_version == last_version: + logger.info("Version %s already tested. No new run needed.", image_version) + ci_lib.add_notification_file( + "nightly-noop", f"Version {image_version} already tested — no-op." + ) + (env.ARTIFACT_DIR / "result.txt").write_text("NO-OP") + return + + logger.info("New version detected: %s (previous: %s)", image_version, last_version or "") + _create_fournos_job(project, image_version, source_cfg) + + result_msg = f"FournosJob created for {project} version {image_version}" + ci_lib.add_notification_file("nightly-triggered", result_msg) + (env.ARTIFACT_DIR / "result.txt").write_text(result_msg) + + +@requires( + cluster="nightly.cluster", + version_key="nightly.version_key", + pipeline="nightly.pipeline", + preset="nightly.preset", + owner="nightly.owner", +) +def _create_fournos_job(project: str, version: str, source_cfg: dict, _cfg) -> None: + """Create a FournosJob for the full test run using the fournos_launcher toolbox.""" + from projects.fournos_launcher.toolbox.submit_and_wait.main import run as submit_and_wait + + namespace = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "fournos-workloads") + + args_list = [_cfg.preset] if _cfg.preset else [] + + overrides = {_cfg.version_key: version} + if mlflow_experiment := source_cfg.get("mlflow_experiment"): + overrides["caliper.export.backend.mlflow.config.experiment"] = mlflow_experiment + logger.info("MLflow experiment override: %s", mlflow_experiment) + + logger.info( + "Creating FournosJob: project=%s, version=%s, cluster=%s, pipeline=%s", + project, + version, + _cfg.cluster, + _cfg.pipeline, + ) + + submit_and_wait( + project, + cluster_name=_cfg.cluster, + args=args_list, + variables_overrides=overrides, + namespace=namespace, + owner=_cfg.owner, + display_name=f"nightly {project} {version}", + pipeline_name=_cfg.pipeline, + exclusive=True, + wait=False, + ) + + +def _write_test_labels() -> None: + """Write ``__test_labels__.yaml`` so the caliper export ``run_naming`` + templates can resolve the ``{outcome}`` placeholder.""" + result_file = env.ARTIFACT_DIR / "result.txt" + if result_file.exists(): + outcome = "noop" if result_file.read_text().strip() == "NO-OP" else "launched" + else: + outcome = "error" + + labels = {"outcome": outcome} + labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + labels_path.write_text(yaml.dump({"labels": labels})) + logger.info("Wrote test labels: %s", labels) + + +def run(): + """Entrypoint for the nightly phase.""" + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + project = config.project.get_config("project.name") + source_cfg = _get_source_config() + logger.info("Project: %s, source config: %s", project, source_cfg) + + try: + image_version = _receive_image(source_cfg) + except Exception as e: + logger.error("receive-image failed: %s", e) + ci_lib.add_notification_file("nightly-receive-image-failed", f"FATAL: {e}") + raise + + try: + _confirm_nightly(project, image_version, source_cfg) + except Exception as e: + logger.error("confirm-nightly failed: %s", e) + ci_lib.add_notification_file("nightly-confirm-failed", f"FATAL: {e}") + raise + finally: + _write_test_labels() diff --git a/projects/core/notifications/README.md b/projects/core/notifications/README.md index cc7442565..e3bf9831b 100644 --- a/projects/core/notifications/README.md +++ b/projects/core/notifications/README.md @@ -59,7 +59,7 @@ success = send_notification( message="🟢 Test completed successfully\nResults: https://example.com/results", github=True, slack=False, - dry_run=False + dry_run=False, ) ``` diff --git a/projects/core/tests/README.md b/projects/core/tests/README.md index 44bbe0541..3f7edcc46 100644 --- a/projects/core/tests/README.md +++ b/projects/core/tests/README.md @@ -62,4 +62,4 @@ If tests fail, check: 1. That FORGE_HOME is properly set or discoverable 2. That expected projects exist with toolbox directories 3. That toolbox commands have `main.py` files with `run` functions -4. That project `__init__.py` files have docstrings \ No newline at end of file +4. That project `__init__.py` files have docstrings diff --git a/projects/foreign_testing/orchestration/foreign_testing.py b/projects/foreign_testing/orchestration/foreign_testing.py index e4451c05d..fd92e891c 100644 --- a/projects/foreign_testing/orchestration/foreign_testing.py +++ b/projects/foreign_testing/orchestration/foreign_testing.py @@ -3,7 +3,7 @@ import pathlib import shutil -from projects.core.ci_entrypoint import run_ci +from projects.core.ci_entrypoint import run_common from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def submit(project_path=None): launch_args = [*["--project-source", str(project_path)], *launch_args] logger.info(f"Submitting deployment with args: {launch_args}") - run_ci.execute_project_operation( + run_common.execute_project_operation( project, operation, launch_args, diff --git a/projects/fournos_launcher/CHANGELOG-orchestration.md b/projects/fournos_launcher/CHANGELOG-orchestration.md index 1430e84d9..e83eb539e 100644 --- a/projects/fournos_launcher/CHANGELOG-orchestration.md +++ b/projects/fournos_launcher/CHANGELOG-orchestration.md @@ -1,5 +1,36 @@ # Fournos Launcher Orchestration Changelog +## 2026-07-13 - Clusterless Mode and New Directives + +### New PR Directives +- **Clusterless Directive**: Added `/clusterless` directive to enable clusterless mode execution + - **Effect**: Sets `fournos.job.exclusive=false` and `fournos.job.clusterless=true` + - **Validation**: Prevents conflicting usage with `/exclusive true` +- **Fournos Environment Directive**: Added `/fournos` directive for namespace targeting + - **Format**: `/fournos wip` or `/fournos staging` + - **Effect**: Sets `fournos.namespace` to `psap-automation-{environment}` + - **Validation**: Restricts to `wip` and `staging` environments only + +### Enhanced Validation +- **Mutual Exclusivity**: Added comprehensive validation to prevent `clusterless=true` and `exclusive=true` conflicts + - **PR Parser Level**: Catches directive conflicts during comment parsing + - **CLI Level**: Validates configuration before command execution + - **Submit Level**: Final validation before job submission +- **Optional Cluster**: Made `/cluster` directive optional when `/clusterless` is used + - **Smart Validation**: Requires cluster specification only when not in clusterless mode + - **Clear Error Messages**: Improved error messaging to explain clusterless option + +### Files Modified +- `pr_args.py` - Added `/clusterless` and `/fournos` directive handlers with validation +- `cli.py` - Enhanced cluster requirement logic and added conflict validation +- `submit.py` - Added clusterless/exclusive mutual exclusivity checks + +### Benefits +- **Flexible Execution**: Enables both cluster-targeted and clusterless job execution modes +- **Environment Targeting**: Simplified namespace selection for different deployment environments +- **Robust Validation**: Prevents configuration conflicts with clear, actionable error messages +- **Backward Compatibility**: Maintains existing directive functionality while adding new capabilities + ## 2026-06-30 - Notification System Integration ### Job Completion Notifications diff --git a/projects/fournos_launcher/CHANGELOG-toolbox.md b/projects/fournos_launcher/CHANGELOG-toolbox.md index e4b0ee2bb..f16dc0ea9 100644 --- a/projects/fournos_launcher/CHANGELOG-toolbox.md +++ b/projects/fournos_launcher/CHANGELOG-toolbox.md @@ -1,5 +1,24 @@ # Fournos Launcher Toolbox Changelog +## 2026-07-13 - Clusterless Mode Support + +### submit_and_wait + +#### Clusterless Execution Support +- **Clusterless Parameter**: Added `clusterless: bool = False` parameter to entrypoint function +- **Template Updates**: Enhanced job.yaml.j2 template to conditionally render `clusterless: true` or `cluster: ` fields +- **Validation Logic**: Added validation to ensure clusterless and exclusive modes are mutually exclusive +- **Optional Cluster**: Made `cluster_name` parameter optional when `clusterless=True` + +#### Files Modified +- `main.py` - Added clusterless parameter support and validation logic +- `templates/job.yaml.j2` - Updated FournosJob template with conditional cluster/clusterless field rendering + +#### Benefits +- Enables clusterless job execution for improved resource flexibility +- Maintains backward compatibility with existing cluster-based workflows +- Provides clear validation for conflicting configuration options + ## 2026-06-24 - Job Management Improvements ### submit_and_wait diff --git a/projects/fournos_launcher/orchestration/cli.py b/projects/fournos_launcher/orchestration/cli.py index 4acd19eea..ff3eda17d 100755 --- a/projects/fournos_launcher/orchestration/cli.py +++ b/projects/fournos_launcher/orchestration/cli.py @@ -53,13 +53,29 @@ def submit(ctx, cluster, project, args, namespace, override, commit): logger.warning(f"Ignoring invalid override format: {override_str} (expected key=value)") # Override config values if provided + clusterless_mode = config.project.get_config("fournos.job.clusterless") + if cluster: config.project.set_config("cluster.name", cluster) else: cluster = config.project.get_config("cluster.name") - if not cluster: - raise ValueError("--cluster or cluster.name is mandatory") - logger.info(f"Using cluster {cluster}") + + if not cluster and not clusterless_mode: + raise ValueError( + "--cluster or cluster.name is mandatory (unless clusterless mode is enabled)" + ) + + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: + raise ValueError( + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" + ) + + if cluster: + logger.info(f"Using cluster {cluster}") + else: + logger.info("Using clusterless mode") if project: config.project.set_config("ci_job.project", project) diff --git a/projects/fournos_launcher/orchestration/config.yaml b/projects/fournos_launcher/orchestration/config.yaml index 4ff135edc..5be5ca092 100644 --- a/projects/fournos_launcher/orchestration/config.yaml +++ b/projects/fournos_launcher/orchestration/config.yaml @@ -25,6 +25,7 @@ fournos: pipeline_name: forge-test-only ci_label: null # will be generated from the CI job UID exclusive: true + clusterless: null env: OPENSHIFT_CI_job: [JOB_TYPE, JOB_NAME, JOB_SPEC, OPENSHIFT_CI, JOB_NAME_SAFE, BUILD_ID] OPENSHIFT_CI_git_pr: [PULL_PULL_SHA, PULL_NUMBER, PULL_BASE_REF, REPO_NAME, REPO_OWNER, PULL_BASE_SHA, JOB_NAME, PULL_TITLE, PULL_REFS, PULL_HEAD_REF] diff --git a/projects/fournos_launcher/orchestration/pr_args.py b/projects/fournos_launcher/orchestration/pr_args.py index b0003acea..270d83dd8 100644 --- a/projects/fournos_launcher/orchestration/pr_args.py +++ b/projects/fournos_launcher/orchestration/pr_args.py @@ -40,6 +40,16 @@ def get_supported_fournos_directives() -> dict[str, str]: Format: /exclusive true|false Example: /exclusive false Effect: Sets fournos.job.exclusive in configuration (default: true).""", + "/clusterless": """Enable clusterless mode for FOURNOS job execution. + Format: /clusterless + Example: /clusterless + Effect: Sets fournos.job.exclusive=false and fournos.job.clusterless=true. + Note: Clusterless and exclusive modes are mutually exclusive.""", + "/fournos": """Set FOURNOS namespace environment. + Format: /fournos environment + Example: /fournos wip + /fournos staging + Effect: Sets fournos.namespace to psap-automation-{environment}.""", "/pipeline": """Set FOURNOS pipeline name for job execution. Format: /pipeline pipeline_name Example: /pipeline llm-load-test @@ -115,6 +125,53 @@ def handle_exclusive_directive(line: str) -> dict[str, str]: return {"fournos.job.exclusive": exclusive_value == "true"} +def handle_clusterless_directive(line: str) -> dict[str, str]: + """ + Handle /clusterless directive for enabling clusterless mode. + + Format: /clusterless + + Args: + line: The directive line + + Returns: + Dictionary with clusterless configuration (sets exclusive=false, clusterless=true) + """ + return {"fournos.job.exclusive": False, "fournos.job.clusterless": True} + + +def handle_fournos_directive(line: str) -> dict[str, str]: + """ + Handle /fournos directive for setting FOURNOS namespace environment. + + Format: /fournos environment + + Args: + line: The directive line + + Returns: + Dictionary with namespace configuration + + Raises: + ValueError: If environment is not wip or staging + """ + FOURNOS_NAMESPACE_BASE = "psap-automation" + VALID_ENVIRONMENTS = {"wip", "staging"} + + environment = line.removeprefix("/fournos ").strip() + + if not environment: + raise ValueError(f"Invalid /fournos directive: environment cannot be empty in '{line}'") + + if environment not in VALID_ENVIRONMENTS: + raise ValueError( + f"Invalid /fournos directive: environment must be one of {VALID_ENVIRONMENTS}, got '{environment}' in '{line}'" + ) + + namespace = f"{FOURNOS_NAMESPACE_BASE}-{environment}" + return {"fournos.namespace": namespace} + + def handle_pipeline_directive(line: str) -> dict[str, str]: """ Handle /pipeline directive for setting pipeline name. @@ -291,6 +348,8 @@ def get_fournos_directive_handlers() -> dict[str, callable]: return { "/cluster": handle_cluster_directive, "/exclusive": handle_exclusive_directive, + "/clusterless": handle_clusterless_directive, + "/fournos": handle_fournos_directive, "/pipeline": handle_pipeline_directive, "/gpu": handle_gpu_directive, "/parallel": handle_parallel_directive, @@ -324,6 +383,14 @@ def parse_fournos_directives( required_directives=None, # No required directives for FOURNOS ) + # Validate directive conflicts + if config_overrides.get("fournos.job.clusterless") and config_overrides.get( + "fournos.job.exclusive" + ): + raise ValueError( + "Conflicting directives: /clusterless and /exclusive true cannot both be used" + ) + # Log successful parses at info level for FOURNOS for directive in parsed_directives: logger.info(f"Parsed FOURNOS directive: {directive}") diff --git a/projects/fournos_launcher/orchestration/submit.py b/projects/fournos_launcher/orchestration/submit.py index dbbcf5944..fbe5ac3b8 100644 --- a/projects/fournos_launcher/orchestration/submit.py +++ b/projects/fournos_launcher/orchestration/submit.py @@ -258,9 +258,18 @@ def submit_job(): # Validate required configuration before job submission cluster_name = config.project.get_config("cluster.name") - if not cluster_name: + clusterless_mode = config.project.get_config("fournos.job.clusterless") + + if not cluster_name and not clusterless_mode: + raise ValueError( + "/cluster (cluster.name) must be set - cannot submit job without target cluster (unless /clusterless is used)" + ) + + # Validate clusterless and exclusive modes are not both enabled + exclusive_mode = config.project.get_config("fournos.job.exclusive") + if clusterless_mode and exclusive_mode: raise ValueError( - "/cluster (cluster.name) must be set - cannot submit job without target cluster" + "Clusterless mode and exclusive mode cannot both be enabled - use /clusterless (sets exclusive=false) or /exclusive false" ) # Get GPU hardware configuration @@ -294,6 +303,7 @@ def submit_job(): "env": env_dict, "ci_label": config.project.get_config("fournos.job.ci_label"), "exclusive": config.project.get_config("fournos.job.exclusive"), + "clusterless": config.project.get_config("fournos.job.clusterless"), "gpu_count": gpu_count, "gpu_type": gpu_type, } diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/main.py b/projects/fournos_launcher/toolbox/submit_and_wait/main.py index 88f544a80..b9bc9e778 100755 --- a/projects/fournos_launcher/toolbox/submit_and_wait/main.py +++ b/projects/fournos_launcher/toolbox/submit_and_wait/main.py @@ -20,6 +20,7 @@ task, template, ) +from projects.core.dsl.control_flow import EarlyReturn from projects.core.dsl.utils.k8s import is_valid_k8s_name, sanitize_k8s_name from projects.core.library import env as env_mod @@ -47,9 +48,9 @@ def __init__( @entrypoint def run( - cluster_name: str, project: str, *, + cluster_name: str | None = None, args: list = None, variables_overrides: dict = None, job_name: str = "", @@ -63,13 +64,15 @@ def run( exclusive: bool = True, gpu_count: int = None, gpu_type: str = None, + clusterless: bool = False, + wait: bool = True, ): """ Submit a FOURNOS job and wait for completion Args: - cluster_name: Name of the target cluster for the FOURNOS job project: The project name to execute (e.g., 'llm_d', 'skeleton') + cluster_name: Name of the target cluster for the FOURNOS job (None when clusterless=True) args: List of arguments to pass to the project (default: empty list) variables_overrides: Dictionary of config variables to override (default: empty dict) job_name: Custom name for the FOURNOS job (auto-generated if empty) @@ -83,12 +86,13 @@ def run( exclusive: Whether the job should run exclusively on its nodes (default: True) gpu_count: Number of GPUs required for the job (default: None) gpu_type: Type of GPU required for the job (default: None) + clusterless: Whether to enable clusterless mode (makes cluster_name optional, forces exclusive=False) (default: False) Examples: # Called by entrypoint with config values: run( + "llm_d", cluster_name="my-cluster", - project="llm_d", args=["test", "--verbose"], variables_overrides={"model": "mistral", "replicas": 2}, namespace="my-fournos-jobs", @@ -101,6 +105,10 @@ def run( gpu_type="nvidia-tesla-v100", ) """ + # Enforce clusterless=True always uses exclusive=False to prevent conflicts + if clusterless: + exclusive = False + # Set defaults if args is None: args = [] @@ -123,8 +131,11 @@ def run( def validate_inputs(args, ctx): """Validate input parameters""" - if not args.cluster_name: - raise ValueError("cluster_name is required") + if not args.cluster_name and not args.clusterless: + raise ValueError("cluster_name is required unless clusterless mode is enabled") + + if args.clusterless and args.exclusive: + raise ValueError("Clusterless mode and exclusive mode cannot both be enabled") if not args.project: raise ValueError("project is required") @@ -199,10 +210,92 @@ def submit_fournos_job(args, ctx): # Apply the job manifest (will raise CalledProcessError with full details on failure) shell.run(f"oc apply -f {ctx.manifest_file}") + if not args.wait: + return EarlyReturn(f"Submitted FournosJob: {ctx.final_job_name} (wait=False)") + return f"Successfully submitted FOURNOS job: {ctx.final_job_name}" -@retry(attempts=360, delay=10, backoff=1.0) +@retry(attempts=30, delay=10, backoff=1.0) +@task +def wait_for_job_to_resolve(args, ctx): + """Wait for FOURNOS job to successfully resolve (reach Pending state)""" + + # Check job status + status_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.phase}}"', + check=False, + ) + + if not status_result.success: + # Check if it's a "not found" error (permanent failure) vs temporary error + if "not found" in status_result.stderr.lower(): + raise FournosJobFailureError( + ctx.final_job_name, + "Job not found during resolve phase - may have been deleted or never created", + args.namespace, + "not_found", + ) + + # Other errors might be temporary, retry + logger.info( + f"Failed to get job status during resolve phase, retrying... (stderr: {status_result.stderr.strip()})" + ) + return False # Retry + + status = status_result.stdout.strip() + + if not status: + logger.info(f"Job {ctx.final_job_name} not processed.") + return False, "Not processed" # Retry + + if status == "Resolving": + logger.info(f"Job {ctx.final_job_name} status: {status}. Still resolving, keep waiting.") + return False, "Resolving" # Retry + + if status == "Stopping": + logger.warning(f"Job {ctx.final_job_name} status: {status}.") + raise FournosJobFailureError( + ctx.final_job_name, "Job stopped in its early stages", args.namespace + ) + + if status == "Failed": + # Get failure details + failure_result = shell.run( + f'oc get fournosjob {ctx.final_job_name} -n {args.namespace} -o jsonpath="{{.status.message}}"', + check=False, + ) + failure_msg = ( + failure_result.stdout.strip() + if failure_result.success + else "Unknown failure during resolve" + ) + raise FournosJobFailureError( + ctx.final_job_name, f"Job failed in its early stages: {failure_msg}", args.namespace + ) + + if status in ["Admitted", "Running", "Succeeded"]: + # Job moved past Pending - this means resolve succeeded + return f"Job {ctx.final_job_name} resolved successfully (current status: {status})" + + if status == "Pending": + return f"Job {ctx.final_job_name} successfully resolved and reached Pending state" + + logger.info(f"Unknown job {ctx.final_job_name} status: {status}") + return False, "Unknown status" # Unknown status, retry + + +@task +def check_early_return(args, ctx): + """Check for early return""" + + if not args.wait: + return EarlyReturn(f"Fournos job successfully launched: {ctx.final_job_name} (wait=False)") + + return f"Fournos job successfully launched: {ctx.final_job_name}" + + +@retry(attempts=3000, delay=30, backoff=1.0) @task def wait_for_job_completion(args, ctx): """Wait for FOURNOS job to complete""" diff --git a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 index 89ab7c5f8..07a84ea9b 100644 --- a/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 +++ b/projects/fournos_launcher/toolbox/submit_and_wait/templates/job.yaml.j2 @@ -10,7 +10,11 @@ metadata: spec: owner: {{ args.owner }} displayName: {{ args.display_name }} + {% if args.clusterless -%} + clusterless: {{ args.clusterless }} + {%- else -%} cluster: {{ args.cluster_name }} + {%- endif %} exclusive: {{ args.exclusive }} pipeline: {{ args.pipeline_name }} {% if args.gpu_count and args.gpu_type -%} diff --git a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py index 4ba749ae8..e4e3420af 100644 --- a/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py +++ b/projects/guidellm/postprocess/guidellm/csv_export/kpi_csv_exporter.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +import logging from pathlib import Path from typing import Any @@ -13,6 +14,8 @@ create_csv_rows_from_kpi_record, ) +logger = logging.getLogger(__name__) + class KPICsvExporter: """Export KPI data to CSV format.""" @@ -62,7 +65,7 @@ def export_kpis_to_csv( rows = create_csv_rows_from_kpi_record(record) csv_rows.extend(rows) except Exception as e: - print(f"Warning: Failed to convert KPI record to CSV row: {e}") + logger.info(f"Warning: Failed to convert KPI record to CSV row: {e}") continue # Write CSV file @@ -90,14 +93,14 @@ def export_kpis_to_csv( # Summary info total_written = len(csv_rows) - print(f"Exported {total_written} KPI records to {output_path}") + logger.info(f"Exported {total_written} KPI records to {output_path}") if skipped_2d_count > 0: - print(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") + logger.info(f"Skipped {skipped_2d_count} 2D KPIs (include_2d_kpis=False)") # Count 2D KPI expansion total_2d_rows = sum(1 for row in csv_rows if getattr(row, "is_2d", False)) if total_2d_rows > 0: - print(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") + logger.info(f"Expanded 2D KPIs into {total_2d_rows} individual data point rows") return str(output_path) diff --git a/projects/guidellm/postprocess/guidellm/parsing/kpis.py b/projects/guidellm/postprocess/guidellm/parsing/kpis.py index 2eb4ac959..35ecf5a85 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/kpis.py +++ b/projects/guidellm/postprocess/guidellm/parsing/kpis.py @@ -13,7 +13,6 @@ LowerBetter, TwoDimensional, build_catalog_from_functions, - create_label_extractor, get_kpi_functions, is_2d_kpi, ) @@ -220,13 +219,34 @@ def guidellm_tpot_p99(unified_record) -> list[tuple[float, float]]: class GuideLLMKpiHandler: """Handles KPI catalog and computation for GuideLLM benchmarks.""" - # Define label extractor for all GuideLLM test conditions - LABEL_EXTRACTOR = create_label_extractor( - { - "strategy": "metrics.strategy", - "duration": "metrics.duration", + # Define custom label extractor for GuideLLM with fallbacks for missing kpi_labels + @staticmethod + def _extract_labels(record) -> dict[str, Any]: + """Extract labels from record with fallbacks for missing kpi_labels.""" + + from projects.caliper.engine.kpi.decorators import _extract_value_by_path + + labels = {} + + # Extract artifact-based labels (graceful handling) + artifact_fields = { + "product_version": "metrics.product_version", + "cluster": "metrics.cluster", + "deployment_profile": "metrics.deployment_profile", + "model_name": "metrics.model_name", + "load_shape": "metrics.benchmark_key", } - ) + + for label_key, path in artifact_fields.items(): + value = _extract_value_by_path(record, path) + if value is not None: + labels[label_key] = str(value) + + labels |= record.metrics.get("kpi_labels", {}) or {} + + return labels + + LABEL_EXTRACTOR = type("TestLabelExtractor", (), {"extract": _extract_labels})() # Metadata fields to include in KPI records but not as labels @staticmethod @@ -284,7 +304,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: # Generate scalar KPIs for each record for r in valid_records: - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -303,9 +322,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if value is None: continue - # Merge base labels, test condition labels, and system labels + # Use only extracted KPI labels, not test labels all_labels = { - **base_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } @@ -338,7 +356,6 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not curves or not request_rates: continue - base_labels = {**r.distinguishing_labels} test_condition_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(r) metadata_fields = GuideLLMKpiHandler.extract_metadata(r) @@ -357,14 +374,8 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: if not value or value is None: continue - # Remove rate-specific labels for aggregated KPIs - aggregated_labels = { - k: v - for k, v in base_labels.items() - if k not in ["concurrency", "rate", "max_concurrency"] - } + # Use only extracted KPI labels, not test labels all_labels = { - **aggregated_labels, **test_condition_labels, "higher_is_better": kpi_func._kpi_higher_is_better, } diff --git a/projects/guidellm/postprocess/guidellm/parsing/parsers.py b/projects/guidellm/postprocess/guidellm/parsing/parsers.py index 4208b2bcf..3853ef99f 100644 --- a/projects/guidellm/postprocess/guidellm/parsing/parsers.py +++ b/projects/guidellm/postprocess/guidellm/parsing/parsers.py @@ -7,6 +7,9 @@ from pathlib import Path from typing import Any +import jsonpath_ng +import yaml + from projects.caliper.engine.model import ( ParseResult, TestBaseNode, @@ -27,6 +30,77 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: return {"facet": "default"} +def _kpi_labels_from_node(node: TestBaseNode) -> dict[str, Any]: + """Extract kpi_labels from a test node.""" + raw = node.labels + kpi_labels = raw.get("kpi_labels") + if isinstance(kpi_labels, dict): + return dict(kpi_labels) + return {} + + +def extract_field_by_jsonpath(data: dict[str, Any], jsonpath: str, default: Any = None) -> Any: + """ + Extract a field from nested dictionary using JSONPath notation. + + Args: + data: The dictionary to extract from + jsonpath: JSONPath like "spec.model.name" or "spec.template.containers[0].name" + default: Default value if field not found + + Returns: + The extracted value or default if not found + + Examples: + extract_field_by_jsonpath(yaml_data, "spec.model.name") + extract_field_by_jsonpath(yaml_data, "spec.template.containers[0].name") + """ + try: + return jsonpath_ng.parse(jsonpath).find(data)[0].value + except IndexError: + return default + + +def parse_product_version_from_annotation(annotation_value: str) -> str | None: + """ + Parse product version from kserve config annotation. + + Args: + annotation_value: Raw annotation like "v3-5-0-ea-2-kserve-config-llm-decode-template" + + Returns: + Cleaned version like "v3.5.0-ea.2" or None if parsing fails + + Examples: + parse_product_version_from_annotation("v3-5-0-ea-2-kserve-config-llm-decode-template") + # Returns: "v3.5.0-ea.2" + """ + if not annotation_value: + return None + + # Remove the kserve suffix + suffix = "-kserve-config-llm-decode-template" + if annotation_value.endswith(suffix): + version_part = annotation_value[: -len(suffix)] + else: + version_part = annotation_value + + # Transform v3-5-0-ea-2 -> v3.5.0-ea.2 + # Replace hyphens with dots in the version numbers, keep -ea- as is + if version_part.startswith("v") and "-ea-" in version_part: + # Split on -ea- to handle the pre-release part separately + base_version, ea_part = version_part.split("-ea-", 1) + + # Replace hyphens with dots in base version (v3-5-0 -> v3.5.0) + base_version = base_version.replace("-", ".") + + # Reconstruct with -ea. format + cleaned_version = f"{base_version}-ea.{ea_part}" + return cleaned_version + + return None + + class GuideLLMParser: """Parser for GuideLLM benchmark JSON artifacts.""" @@ -36,6 +110,99 @@ def _is_benchmarks_artifact(path: Path) -> bool: path.name.startswith("benchmarks-rate-") and path.suffix == ".json" ) + @staticmethod + def _is_llmisvc_artifact(path: Path) -> bool: + """Check if path is an LLMInferenceService YAML artifact.""" + return path.name in [ + "llminferenceservice.yaml", + "llminferenceservice.yml", + ] and "__capture_llmisvc_state" in str(path) + + @staticmethod + def _is_config_artifact(path: Path) -> bool: + """Check if path is a config.yaml artifact.""" + return path.name == "config.yaml" + + def extract_fields_from_llmisvc(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from LLMInferenceService YAML file. + + Args: + file_path: Path to llminferenceservice.yaml file + + Returns: + Dictionary with extracted fields (product_version, deployment_profile, model_name) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract product version from kserve annotation + annotation_value = extract_field_by_jsonpath( + yaml_data, 'status.annotations["serving.kserve.io/config-llm-decode-template"]' + ) + if annotation_value: + product_version = parse_product_version_from_annotation(annotation_value) + if product_version: + result["product_version"] = product_version + logging.info(f"Extracted product_version '{product_version}' from {file_path}") + + # Extract deployment profile from forge annotation + deployment_profile = extract_field_by_jsonpath( + yaml_data, 'metadata.annotations["forge.openshift.io/deployment-profile"]' + ) + if deployment_profile: + result["deployment_profile"] = deployment_profile + logging.info( + f"Extracted deployment_profile '{deployment_profile}' from {file_path}" + ) + + # Extract model name from spec + model_name = extract_field_by_jsonpath(yaml_data, "spec.model.name") + if model_name: + result["model_name"] = model_name + logging.info(f"Extracted model_name '{model_name}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + + def extract_fields_from_config(self, file_path: Path) -> dict[str, str]: + """ + Extract multiple fields from config.yaml file. + + Args: + file_path: Path to config.yaml file + + Returns: + Dictionary with extracted fields (cluster, benchmark_key) + """ + result = {} + try: + yaml_data = yaml.safe_load(file_path.read_text(encoding="utf-8")) + if not isinstance(yaml_data, dict): + return result + + # Extract cluster name + cluster = extract_field_by_jsonpath(yaml_data, "ci_job.cluster") + if cluster: + result["cluster"] = cluster + logging.info(f"Extracted cluster '{cluster}' from {file_path}") + + # Extract benchmark key + benchmark_key = extract_field_by_jsonpath(yaml_data, "runtime.benchmark_key") + if benchmark_key: + result["benchmark_key"] = benchmark_key + logging.info(f"Extracted benchmark_key '{benchmark_key}' from {file_path}") + + except Exception as e: + logging.warning(f"Failed to extract fields from {file_path}: {e}") + + return result + def parse_benchmarks_json( self, file_path: Path ) -> tuple[list[GuideLLMBenchmark], GuideLLMConfiguration | None, list[str]]: @@ -376,6 +543,10 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: benchmarks_files = [p for p in node.artifact_paths if self._is_benchmarks_artifact(p)] benchmarks_files.sort(key=lambda path: path.name) + # Also look for LLMInferenceService YAML files and config.yaml files for system information + llmisvc_files = [p for p in node.artifact_paths if self._is_llmisvc_artifact(p)] + config_files = [p for p in node.artifact_paths if self._is_config_artifact(p)] + if not benchmarks_files: # No benchmark result JSON found for this node, create empty record labels = _labels_from_node(node) @@ -408,6 +579,25 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: if node_config: metrics["configuration"] = node_config.to_dict() + # Extract fields from LLMInferenceService YAML if available + for llmisvc_file in llmisvc_files: + llmisvc_fields = self.extract_fields_from_llmisvc(llmisvc_file) + for field_name, field_value in llmisvc_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract fields from config.yaml if available + for config_file in config_files: + config_fields = self.extract_fields_from_config(config_file) + for field_name, field_value in config_fields.items(): + if field_value and field_name not in metrics: + metrics[field_name] = field_value + + # Extract kpi_labels from test labels + kpi_labels = _kpi_labels_from_node(node) + if kpi_labels: + metrics["kpi_labels"] = kpi_labels + records.append( UnifiedResultRecord( test_base_path=str(node.test_path), diff --git a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py index 98534c9f3..8eaf0007e 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py +++ b/projects/guidellm/postprocess/guidellm/plotting/kpi_report.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from html import escape from pathlib import Path from typing import Any @@ -10,6 +11,8 @@ create_report_filename, ) +logger = logging.getLogger(__name__) + def generate_kpi_report( records: list[Any], @@ -35,7 +38,7 @@ def generate_kpi_report( # Handle multiple records by creating sections for each test if len(records) > 1: - print(f"📊 Generating multi-test KPI report for {len(records)} test records") + logger.info(f"📊 Generating multi-test KPI report for {len(records)} test records") return _generate_multi_test_kpi_report(records, output_dir, report_title, report_number) # Use the single record to extract test conditions and KPIs @@ -460,7 +463,7 @@ def _generate_multi_test_kpi_report( test_sections = [] for i, record in enumerate(records, 1): - print(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") + logger.info(f" 📋 Processing test {i}/{len(records)}: {record.test_base_path}") # Extract test condition labels test_labels = GuideLLMKpiHandler.LABEL_EXTRACTOR.extract(record) diff --git a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py index 549c0783c..6e8ef1c78 100644 --- a/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py +++ b/projects/guidellm/postprocess/guidellm/plotting/performance_analysis.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -18,6 +19,8 @@ save_figure, ) +logger = logging.getLogger(__name__) + def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: Any = 0.0) -> Any: """ @@ -51,18 +54,18 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data Returns: DataFrame with all benchmark metrics and distinguishing labels """ - print(f"📊 Converting {len(records)} records to analysis dataframe...") + logger.info(f"📊 Converting {len(records)} records to analysis dataframe...") data = [] # Get parameters that vary across all records for legend names - print("🔍 Analyzing parameter variations for meaningful legend names...") + logger.info("🔍 Analyzing parameter variations for meaningful legend names...") varying_params = get_varying_parameters(records) if varying_params: - print(f" Found varying parameters: {', '.join(sorted(varying_params))}") + logger.info(f" Found varying parameters: {', '.join(sorted(varying_params))}") else: - print(" No varying parameters found - using default naming") + logger.info(" No varying parameters found - using default naming") - print("📝 Processing records and extracting metrics...") + logger.info("📝 Processing records and extracting metrics...") for record in records: # Skip records without GuideLLM data or missing benchmarks if not record.run_identity.get("guidellm") or record.metrics.get("no_benchmarks_found"): @@ -77,7 +80,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if has_curves and isinstance(request_rates, list) and len(request_rates) > 0: # New format: expand performance curves into multiple data points - print( + logger.info( f" 🔄 Expanding performance curves for {legend_name} ({len(request_rates)} points)" ) curves = record.metrics.get("performance_curves", {}) @@ -157,7 +160,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) else: # Old format: single data point per record (backward compatibility) - print(f" 📊 Using scalar metrics for {legend_name}") + logger.info(f" 📊 Using scalar metrics for {legend_name}") # Extract scalar request_rate if it's a single value request_rate = ( request_rates[0] @@ -226,14 +229,14 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data data.append(row) if not data: - print("⚠️ No GuideLLM data found in records") + logger.info("⚠️ No GuideLLM data found in records") return pd.DataFrame() - print(f"✅ Successfully processed {len(data)} data points from {len(records)} records") + logger.info(f"✅ Successfully processed {len(data)} data points from {len(records)} records") df = pd.DataFrame(data) # Sort for consistent ordering - print("📋 Organizing data by configuration, concurrency, and request rate...") + logger.info("📋 Organizing data by configuration, concurrency, and request rate...") # Sort and fill any NaN values in numeric columns with 0 for consistent plotting df = df.sort_values( ["test_configuration", "request_concurrency", "request_rate", "rate_point_index"] @@ -259,7 +262,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data ] ) - print( + logger.info( f"🎯 Expanded {total_records} benchmark records into {len(df)} data points across {len(configs)} configurations:" ) for config in configs: @@ -270,7 +273,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if rate_points > 1 else f"{config_data['request_rate'].iloc[0]:.1f} req/s" ) - print(f" • {config}: {rate_points} rate points ({rate_range})") + logger.info(f" • {config}: {rate_points} rate points ({rate_range})") return df @@ -279,11 +282,11 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): """Create throughput scaling scatter plot.""" try: - print("📈 Creating throughput scaling scatter plot...") + logger.info("📈 Creating throughput scaling scatter plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for throughput scaling plot") + logger.info("⚠️ No data available for throughput scaling plot") return None title = f"Request Throughput vs Concurrency by Configuration{title_context}" @@ -314,22 +317,22 @@ def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ Throughput scaling plot created successfully") + logger.info("✅ Throughput scaling plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create throughput scaling plot: {e}") + logger.info(f"❌ Failed to create throughput scaling plot: {e}") return None def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = ""): """Create latency vs throughput scatter plot.""" try: - print("📈 Creating latency vs throughput trade-off plot...") + logger.info("📈 Creating latency vs throughput trade-off plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for latency vs throughput plot") + logger.info("⚠️ No data available for latency vs throughput plot") return None title = f"Latency vs Throughput Trade-off{title_context}" @@ -353,22 +356,22 @@ def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = "") showlegend=True, width=800, height=500, font={"size": 12}, title_font_size=16 ) - print("✅ Latency vs throughput plot created successfully") + logger.info("✅ Latency vs throughput plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create latency vs throughput plot: {e}") + logger.info(f"❌ Failed to create latency vs throughput plot: {e}") return None def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput vs concurrency line plot.""" try: - print("📈 Creating token throughput vs concurrency scaling plot...") + logger.info("📈 Creating token throughput vs concurrency scaling plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for token throughput vs concurrency plot") + logger.info("⚠️ No data available for token throughput vs concurrency plot") return None title = f"Token Throughput vs Concurrency{title_context}
Higher is better" @@ -399,22 +402,22 @@ def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput vs concurrency plot created successfully") + logger.info("✅ Token throughput vs concurrency plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput vs concurrency plot: {e}") + logger.info(f"❌ Failed to create token throughput vs concurrency plot: {e}") return None def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): """Create TTFT analysis line plot.""" try: - print("📈 Creating TTFT (Time to First Token) analysis plot...") + logger.info("📈 Creating TTFT (Time to First Token) analysis plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for TTFT analysis plot") + logger.info("⚠️ No data available for TTFT analysis plot") return None title = f"TTFT vs Concurrency{title_context}
Lower is better" @@ -440,23 +443,23 @@ def create_ttft_analysis_plot(df: pd.DataFrame, title_context: str = ""): ) fig.update_yaxes(rangemode="tozero") - print("✅ TTFT analysis plot created successfully") + logger.info("✅ TTFT analysis plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create TTFT analysis plot: {e}") + logger.info(f"❌ Failed to create TTFT analysis plot: {e}") return None def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput percentiles plot.""" try: - print("📈 Creating token throughput percentiles distribution plot...") + logger.info("📈 Creating token throughput percentiles distribution plot...") import plotly.express as px import plotly.graph_objects as go if df.empty: - print("⚠️ No data available for token throughput percentiles plot") + logger.info("⚠️ No data available for token throughput percentiles plot") return None title = f"Output Token Throughput Percentiles{title_context}
Higher is better" @@ -465,7 +468,9 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st # Get unique configurations and colors configurations = sorted(df["test_configuration"].unique()) - print(f" Plotting {len(configurations)} configurations with percentile distributions...") + logger.info( + f" Plotting {len(configurations)} configurations with percentile distributions..." + ) available_colors = px.colors.qualitative.Set1 color_map = { config: available_colors[i % len(available_colors)] @@ -480,7 +485,7 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ("P75", "output_tokens_per_second_p75", {"width": 3, "dash": "dash"}, 0.9), ("P90", "output_tokens_per_second_p90", {"width": 2, "dash": "dashdot"}, 0.8), ] - print(f" Adding {len(percentiles)} percentile lines per configuration...") + logger.info(f" Adding {len(percentiles)} percentile lines per configuration...") for config in configurations: config_df = df[df["test_configuration"] == config].sort_values("request_concurrency") @@ -510,11 +515,11 @@ def create_token_throughput_percentiles_plot(df: pd.DataFrame, title_context: st ) fig.update_yaxes(rangemode="tozero") - print("✅ Token throughput percentiles plot created successfully") + logger.info("✅ Token throughput percentiles plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create token throughput percentiles plot: {e}") + logger.info(f"❌ Failed to create token throughput percentiles plot: {e}") return None @@ -565,7 +570,7 @@ def generate_token_throughput_vs_concurrency( report_number: int | None = None, ) -> str | None: """Generate token throughput vs concurrency analysis and save to file.""" - print("\n🚀 Generating token throughput vs concurrency analysis...") + logger.info("\n🚀 Generating token throughput vs concurrency analysis...") df = create_dataframe_from_records(records) if df.empty: return None @@ -632,7 +637,7 @@ def generate_comprehensive_performance_report( report_title: str = "GuideLLM Performance Analysis", ) -> str | None: """ - Generate comprehensive performance analysis report with separate plot files. + Generate comprehensive performance analysis report with separate plots for each loadshape. Args: records: List of unified result records @@ -645,16 +650,24 @@ def generate_comprehensive_performance_report( # Create report identifier using core utilities display_title = create_report_title_display(report_title, report_number) - print(f"\n🚀 Starting {display_title} generation...") - print("=" * 70) + logger.info(f"\n🚀 Starting {display_title} generation...") + logger.info("=" * 70) - df = create_dataframe_from_records(records) - if df.empty: - print("❌ No data available for analysis") + # Group records by guidellm_loadshape + loadshape_groups = {} + for record in records: + loadshape = record.distinguishing_labels.get("guidellm_loadshape", "default") + if loadshape not in loadshape_groups: + loadshape_groups[loadshape] = [] + loadshape_groups[loadshape].append(record) + + if not loadshape_groups: + logger.info("❌ No data available for analysis") return None - print("\n📊 Generating performance analysis plots...") - print(" This may take a moment as we create high-quality visualizations...") + logger.info( + f"\n🔍 Found {len(loadshape_groups)} loadshape(s): {', '.join(loadshape_groups.keys())}" + ) # Generate all the plots as figures plot_functions = [ @@ -673,89 +686,354 @@ def generate_comprehensive_performance_report( report_dir = output_dir / report_dir_name report_dir.mkdir(exist_ok=True) - print(f"\n📁 Created report directory: {report_dir_name}") - - plots_data = [] - for i, (plot_name, plot_func) in enumerate(plot_functions, 1): - print(f"\n📈 [{i}/{len(plot_functions)}] Processing {plot_name}...") - try: - fig = plot_func(df, title_context) - if fig: - # Save as both PNG and HTML in the report directory - filename = plot_name.lower().replace(" ", "_") - print(f"💾 Saving {plot_name} as PNG and HTML...") - - # Save PNG image - width = 900 if "Percentiles" in plot_name else 800 - height = 600 if "Percentiles" in plot_name else 500 - png_path = save_figure( - fig, report_dir, filename, as_image=True, width=width, height=height - ) - - # Save HTML version - html_path = save_figure(fig, report_dir, filename, as_image=False) + logger.info(f"\n📁 Created report directory: {report_dir_name}") + + all_plots_data = [] + + # Process each loadshape separately + for loadshape, loadshape_records in loadshape_groups.items(): + logger.info(f"\n📊 Processing loadshape: {loadshape}") + logger.info(f" Records: {len(loadshape_records)}") + + # Create DataFrame for this loadshape + df = create_dataframe_from_records(loadshape_records) + if df.empty: + logger.info(f" ⚠️ No data available for loadshape: {loadshape}") + continue + + logger.info(f" 📈 Generating {len(plot_functions)} plots for {loadshape}...") + + # Create subdirectory for this loadshape + loadshape_dir = report_dir / loadshape + loadshape_dir.mkdir(exist_ok=True) + + # Extract model information for this loadshape + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + loadshape_plots = [] + for i, (plot_name, plot_func) in enumerate(plot_functions, 1): + logger.info( + f" 📊 [{i}/{len(plot_functions)}] Creating {plot_name} for {loadshape}..." + ) + try: + # Create subtitle with model and loadshape info + subtitle = f"Model: {model_info} • Load Shape: {loadshape}" + + # Add original title context if provided + if title_context: + loadshape_title_context = f"{title_context}
{subtitle}" + else: + loadshape_title_context = f"
{subtitle}" + + fig = plot_func(df, loadshape_title_context) + if fig: + # Save as both PNG and HTML in the loadshape directory + filename = f"{loadshape}_{plot_name.lower().replace(' ', '_')}" + + # Save PNG image + width = 900 if "Percentiles" in plot_name else 800 + height = 600 if "Percentiles" in plot_name else 500 + png_path = save_figure( + fig, loadshape_dir, filename, as_image=True, width=width, height=height + ) - if png_path and html_path: - # Store relative paths for linking - plots_data.append( - ( - plot_name, - f"{report_dir_name}/{Path(png_path).name}", # PNG path - f"{report_dir_name}/{Path(html_path).name}", # HTML path + # Save HTML version + html_path = save_figure(fig, loadshape_dir, filename, as_image=False) + + if png_path and html_path: + # Store relative paths for linking + loadshape_plots.append( + ( + plot_name, + f"{report_dir_name}/{loadshape}/{Path(png_path).name}", # PNG path + f"{report_dir_name}/{loadshape}/{Path(html_path).name}", # HTML path + ) ) + logger.info(f" ✅ {plot_name} saved for {loadshape}") + else: + logger.info( + f" ⚠️ {plot_name} could not be created for {loadshape} (no figure returned)" ) - print(f"✅ {plot_name} saved as PNG and HTML") - else: - print(f"⚠️ {plot_name} could not be created (no figure returned)") - except Exception as e: - print(f"❌ Failed to generate {plot_name}: {e}") + except Exception as e: + logger.info(f" ❌ Failed to generate {plot_name} for {loadshape}: {e}") - print(f"\n✅ Successfully generated {len(plots_data)} visualizations!") + # Store loadshape plots data + if loadshape_plots: + all_plots_data.append((loadshape, loadshape_plots)) - # Generate summary statistics - print("\n📊 Computing performance statistics and insights...") - summary_stats = _generate_performance_summary(df) + if not all_plots_data: + logger.info("❌ No plots were successfully generated") + return None - if summary_stats.get("best_tokens"): - best = summary_stats["best_tokens"] - print(f" 🏆 Best performance: {best['value']:.0f} tok/s ({best['config']})") + logger.info(f"\n✅ Successfully generated plots for {len(all_plots_data)} loadshape(s)!") - # Create comprehensive HTML report - print("\n📝 Assembling comprehensive HTML report...") - print(" 🔗 Creating report with images linking to interactive HTML versions...") + # Create comprehensive HTML report with sections for each loadshape + logger.info("\n📝 Assembling comprehensive HTML report...") + logger.info(" 🔗 Creating report with loadshape sections and interactive HTML links...") - html_content = _create_comprehensive_html_report_with_images( - plots_data, summary_stats, title_context, display_title - ) + html_content = f""" + + + + + + {display_title} + + + + + +

Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.

+ +
+

{display_title}

+

Generated performance analysis with separate visualizations for each loadshape

+
+ +
+

Quick Navigation:

+
    """ + + # Add navigation links for each loadshape + for loadshape, _ in all_plots_data: + html_content += f'\n
  • {loadshape}
    • ' + + html_content += """ +
+
+""" + + # Add sections for each loadshape with tabbed interface + container_id = 0 + for loadshape, plots in all_plots_data: + # Extract model information for this loadshape + loadshape_records = loadshape_groups[loadshape] + model_info = "Unknown" + if loadshape_records: + # Try to get model information from distinguishing labels + first_record = loadshape_records[0] + model_info = ( + first_record.distinguishing_labels.get("model") + or first_record.distinguishing_labels.get("model_name") + or first_record.distinguishing_labels.get("llm_model") + or first_record.run_identity.get("model") + or "Unknown" + ) + + html_content += f""" +
+

Loadshape: {loadshape}

+

+ Model: {model_info} • Load Shape: {loadshape} +

+ +
+
""" + + # Define tab mapping with icons + tab_mapping = { + "Token Throughput vs Concurrency": "🚀 Throughput", + "TTFT Analysis": "⏱️ TTFT", + "Token Throughput Percentiles": "📊 Throughput Percentiles", + "Throughput Scaling": "📈 Scaling", + "Latency vs Throughput": "⚖️ Latency Trade-off", + } + + # Create tab headers + for tab_idx, (plot_name, _png_path, _html_path) in enumerate(plots): + tab_title = tab_mapping.get(plot_name, plot_name) + active_class = " active" if tab_idx == 0 else "" + html_content += f""" + """ + + html_content += """ +
""" + + # Create tab contents + for tab_idx, (plot_name, png_path, html_path) in enumerate(plots): + active_class = " active" if tab_idx == 0 else "" + + # Create descriptions for each plot type + descriptions = { + "Token Throughput vs Concurrency": "Token generation throughput scaling analysis across different concurrency levels.", + "TTFT Analysis": "Time To First Token analysis - measuring responsiveness and initial latency.", + "Token Throughput Percentiles": "Complete token throughput percentile distribution analysis.", + "Throughput Scaling": "Throughput scaling behavior and efficiency analysis.", + "Latency vs Throughput": "Trade-off analysis between latency and throughput performance.", + } + + description = descriptions.get(plot_name, f"{plot_name} performance analysis.") + + html_content += f""" +
+
+

{plot_name}

+

{description}

+

{plot_name}

+
+
""" + + html_content += """ +
+
""" + container_id += 1 + + html_content += """ + +""" + + # Save the main HTML report + main_html_filename = create_report_filename( + report_title.lower().replace(" ", "_"), report_number, report_title, "html" ) - report_file = output_dir / filename - print(f"\n💾 Writing {display_title} to: {report_file.name}") - report_file.write_text(html_content, encoding="utf-8") + main_html_path = output_dir / main_html_filename + + with open(main_html_path, "w", encoding="utf-8") as f: + f.write(html_content) + + logger.info(f"✅ Comprehensive report saved as: {main_html_filename}") + logger.info(f"📁 Individual plots organized in subdirectories under: {report_dir_name}/") - file_size_mb = report_file.stat().st_size / (1024 * 1024) - print(f"📄 Report complete! File size: {file_size_mb:.1f}MB") - print("=" * 70) - print(f"🎉 {display_title} ready: {report_file.name}") + logger.info("=" * 70) + logger.info(f"🎉 {display_title} ready: {main_html_path.name}") - return str(report_file) + return str(main_html_path) except Exception as e: - print(f"❌ Failed to generate comprehensive performance report: {e}") + logger.info(f"❌ Failed to generate comprehensive performance report: {e}") return None def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: """Generate performance summary statistics from the dataframe.""" if df.empty: - print("⚠️ No data available for performance summary") + logger.info("⚠️ No data available for performance summary") return {} - print(" 🔍 Analyzing best performers across all metrics...") + logger.info(" 🔍 Analyzing best performers across all metrics...") # Find best performers best_tokens_idx = df["tokens_per_second"].idxmax() @@ -765,7 +1043,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: best_ttft_idx = df["ttft_median_ms"].idxmin() # Configuration analysis - print(" 📊 Computing configuration performance rankings...") + logger.info(" 📊 Computing configuration performance rankings...") config_performance = ( df.groupby("test_configuration") .agg( @@ -779,7 +1057,7 @@ def _generate_performance_summary(df: pd.DataFrame) -> dict[str, Any]: .sort_values("tokens_per_second", ascending=False) ) - print(f" ✅ Performance analysis complete for {len(config_performance)} configurations") + logger.info(f" ✅ Performance analysis complete for {len(config_performance)} configurations") return { "total_configurations": len(df["test_configuration"].unique()), diff --git a/projects/guidellm/postprocess/guidellm/plugin.py b/projects/guidellm/postprocess/guidellm/plugin.py index dfe8ff3ed..f570050df 100644 --- a/projects/guidellm/postprocess/guidellm/plugin.py +++ b/projects/guidellm/postprocess/guidellm/plugin.py @@ -145,9 +145,12 @@ def visualize( return paths # Generate reports using the registry + invalid_reports: list[str] = [] + for report_name in wanted: if report_name not in PLOT_REGISTRY: logger.warning("Unknown report '%s' requested", report_name) + invalid_reports.append(report_name) continue plot_config = PLOT_REGISTRY[report_name] @@ -165,6 +168,14 @@ def visualize( except Exception as e: logger.error("Error generating %s: %s", report_name, e) + # Check for invalid report names and raise error if found + if invalid_reports: + available_reports = list(PLOT_REGISTRY.keys()) + raise ValueError( + f"Invalid report types requested: {sorted(invalid_reports)}. " + f"Available types in this plugin: {sorted(available_reports)}" + ) + return paths def kpi_catalog(self) -> list[dict[str, Any]]: @@ -175,6 +186,27 @@ def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: """Compute KPI values from the unified model.""" return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to CSV format using GuideLLM's CSV exporter. + + Args: + kpi_records: KPI records from compute_kpis() + output_path: Path where to write the CSV file + include_header_comments: Whether to include descriptive header comments + + Returns: + Path to the generated CSV file + """ + from .csv_export import KPICsvExporter + + exporter = KPICsvExporter() + return exporter.export_kpis_to_csv(kpi_records, output_path, include_header_comments) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: """Build AI evaluation payload from the unified model.""" return self.ai_evaluator.build_payload(model, self) diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py index 91cbcc445..65cafe544 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/main.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/main.py @@ -2,6 +2,8 @@ from __future__ import annotations +import gzip +import json import logging import subprocess from pathlib import Path @@ -23,6 +25,23 @@ logger = logging.getLogger(__name__) +def trim_benchmark_json(obj): + """Remove 'requests' field recursively from JSON data""" + if isinstance(obj, dict): + return { + k: trim_benchmark_json(v) + for k, v in obj.items() + if k + not in [ + "requests", + ] + } + elif isinstance(obj, list): + return [trim_benchmark_json(item) for item in obj] + else: + return obj + + @entrypoint def run( *, @@ -33,6 +52,9 @@ def run( timeout: int = 900, pvc_size: str = "1Gi", guidellm_args: list[str] | None = None, + hf_token_secret: str = "", + fs_group: int | None = None, + keep_full_benchmark_file: bool = False, ) -> int: """ Run the GuideLLM benchmark against a resolved endpoint. @@ -45,6 +67,9 @@ def run( timeout: Timeout in seconds to wait for job completion pvc_size: Size of the PersistentVolumeClaim for storing results guidellm_args: List of additional guidellm arguments (e.g., ["--rate=10", "--max-seconds=30"]) + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds securityContext.fsGroup to the GuideLLM job pod. + keep_full_benchmark_file: Whether to keep the full untrimmed benchmark JSON files alongside trimmed ones (default: False) """ execute_tasks(locals()) @@ -118,19 +143,12 @@ def _best_effort_delete(description: str, *oc_args: str) -> None: @task def create_guidellm_resources_task(args, ctx): - """Create the GuideLLM benchmark PVC and job""" + """Create the GuideLLM benchmark job and PVC with job as owner""" # Ensure src directory exists (args.artifact_dir / "src").mkdir(parents=True, exist_ok=True) - oc_apply( - args.artifact_dir / "src" / "guidellm-pvc.yaml", - render_guidellm_pvc_from_parts( - namespace=ctx.target_namespace, - name=ctx.benchmark_name, - pvc_size=args.pvc_size, - ), - ) + # Create the job first oc_apply( args.artifact_dir / "src" / "guidellm-job.yaml", render_guidellm_job_from_parts( @@ -139,12 +157,39 @@ def create_guidellm_resources_task(args, ctx): image=ctx.image, endpoint_url=args.endpoint_url, guidellm_args=ctx.guidellm_args, + hf_token_secret=args.hf_token_secret, + fs_group=args.fs_group, ), ) - return f"GuideLLM benchmark {ctx.benchmark_name} created" + # Get the job metadata for owner reference + job_data = oc_get_json("job", name=ctx.benchmark_name, namespace=ctx.target_namespace) + + # Create owner reference from job metadata + owner_reference = { + "apiVersion": "batch/v1", + "kind": "Job", + "name": job_data["metadata"]["name"], + "uid": job_data["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } -@retry(attempts=180, delay=10, backoff=1.0) + # Create the PVC with job as owner + oc_apply( + args.artifact_dir / "src" / "guidellm-pvc.yaml", + render_guidellm_pvc_from_parts( + namespace=ctx.target_namespace, + name=ctx.benchmark_name, + pvc_size=args.pvc_size, + owner_reference=owner_reference, + ), + ) + + return f"GuideLLM benchmark {ctx.benchmark_name} created with job as PVC owner" + + +@retry(attempts=360, delay=30, backoff=1.0) @task def wait_guidellm_benchmark_task(args, ctx): """Wait for the GuideLLM benchmark job to complete""" @@ -264,7 +309,7 @@ def create_copy_pod(args, ctx): return f"Created copy pod {ctx.benchmark_name}-copy" -@retry(attempts=24, delay=5, backoff=1.0) +@retry(attempts=60, delay=5, backoff=1.0) @task def wait_copy_pod_ready(args, ctx): """Wait for copy pod to be ready""" @@ -288,6 +333,7 @@ def extract_results(args, ctx): """Extract GuideLLM results from copy pod""" results_dir = args.artifact_dir / "artifacts" / "results" + results_dir.mkdir(parents=True, exist_ok=True) extracted_files: list[dict[str, str | None]] = [] for run in ctx.guidellm_runs: if run.rate is None: @@ -297,21 +343,57 @@ def extract_results(args, ctx): remote_path = f"/results/benchmarks-{run.label}.json" local_path = results_dir / f"benchmarks-{run.label}.json" + logger.info(f"Retrieving the compressed benchmark file for {run.label}...") + + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") + result = oc( "exec", "-n", ctx.target_namespace, f"{ctx.benchmark_name}-copy", "--", - "cat", + "gzip", + "-c", remote_path, check=False, log_stdout=False, + stdout_dest=temp_gz_path, + text=False, ) - if result.returncode != 0 or not result.stdout: + if result.returncode != 0: raise RuntimeError(f"No results found for {ctx.benchmark_name} run {run.label}") - write_text(local_path, result.stdout) + # Extract and parse JSON from local compressed file + logger.info(f"Extracting compressed benchmark file for {run.label}...") + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + raise RuntimeError( + f"Failed to extract compressed results for run {run.label}: {e}" + ) from e + + # Always create trimmed version + logger.info(f"Trimming the benchmark.json file for {run.label}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON for {run.label}, writing raw data") + write_text(local_path, json_content) + + # Manage full benchmark file based on flag + if not args.keep_full_benchmark_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full benchmark file {temp_gz_path.name}") + else: + logger.info(f"Keeping full benchmark file {temp_gz_path.name}") extracted_files.append( { "label": run.label, @@ -332,6 +414,94 @@ def extract_results(args, ctx): return f"Extracted results for {ctx.benchmark_name}" +def _copy_result_file( + namespace: str, + pod: str, + remote_path: str, + local_path: Path, + *, + trim_json: bool = True, + keep_full_file: bool = False, +) -> Path | None: + """Copy a result file from a pod, with optional compression and trimming. + + Args: + namespace: Kubernetes namespace + pod: Pod name + remote_path: Path to file inside the pod + local_path: Local destination path + trim_json: Whether to trim large fields from JSON files (default: True) + keep_full_file: Whether to keep the full untrimmed file alongside trimmed one (default: False) + + Returns: + Path to the copied file, or None if copy failed + """ + import gzip + + # Use compression approach for better handling of large files + logger.info(f"Retrieving compressed file from {remote_path}...") + + # Save compressed version to temp file first + temp_gz_path = local_path.with_suffix(".json.gz") + + # Compress and stream directly from pod + result = oc( + "exec", + "-n", + namespace, + pod, + "--", + "gzip", + "-c", + remote_path, + check=False, + log_stdout=False, + stdout_dest=temp_gz_path, + text=False, + ) + if result.returncode != 0: + logging.getLogger(__name__).warning("gzip extraction failed (rc=%d)", result.returncode) + return None + + # Extract and parse JSON from local compressed file + try: + with gzip.open(temp_gz_path, "rt", encoding="utf-8") as f: + json_content = f.read() + except Exception as e: + logging.getLogger(__name__).warning("Failed to extract compressed results: %s", e) + temp_gz_path.unlink(missing_ok=True) + return None + + # Create trimmed version if requested and file is JSON + if trim_json and local_path.suffix.lower() == ".json": + logger.info(f"Trimming JSON file {local_path.name}...") + try: + raw_data = json.loads(json_content) + cleaned_data = trim_benchmark_json(raw_data) + with open(local_path, "w") as f: + json.dump(cleaned_data, f, separators=(",", ":")) + except json.JSONDecodeError: + logger.warning(f"Failed to parse JSON, writing raw data to {local_path}") + write_text(local_path, json_content) + else: + # Write raw content for non-JSON files or when trimming is disabled + write_text(local_path, json_content) + + # Manage full file based on keep_full_file flag + if not keep_full_file: + # Remove the compressed full file + if temp_gz_path.exists(): + temp_gz_path.unlink() + logger.info(f"Removed full file {temp_gz_path.name}") + else: + logger.info(f"Keeping full file {temp_gz_path.name}") + + if local_path.exists() and local_path.stat().st_size > 0: + return local_path + + return None + + @task def cleanup_copy_pod(args, ctx): """Delete the copy pod after results extraction""" @@ -393,19 +563,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name ) # Capture job logs - result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job.logs", ) - if result.returncode == 0 and result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job.logs", result.stdout) # Capture additional debugging info - pods_result = oc( + oc( "get", "pods", "-n", @@ -415,11 +584,10 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_pods.yaml", ) - if pods_result.returncode == 0 and pods_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_pods.yaml", pods_result.stdout) - job_result = oc( + oc( "get", "job", benchmark_name, @@ -428,20 +596,18 @@ def capture_guidellm_state(*, artifact_dir: Path, namespace: str, benchmark_name "-oyaml", check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_detailed.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_detailed.yaml", job_result.stdout) - logs_result = oc( + oc( "logs", f"job/{benchmark_name}", "-n", namespace, check=False, log_stdout=False, + stdout_dest=artifacts_dir / "guidellm_benchmark_job_logs.txt", ) - if logs_result.returncode == 0 and logs_result.stdout: - write_text(artifacts_dir / "guidellm_benchmark_job_logs.txt", logs_result.stdout) def capture_get( @@ -460,9 +626,7 @@ def capture_get( if selector: args.extend(["-l", selector]) args.extend(["-o", output]) - result = oc(*args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + oc(*args, check=False, log_stdout=False, stdout_dest=destination) if __name__ == "__main__": diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 index 8324631c4..10b40fe0a 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/templates/guidellm_job.yaml.j2 @@ -17,6 +17,10 @@ spec: spec: serviceAccountName: default restartPolicy: Never +{% if fs_group is not none %} + securityContext: + fsGroup: {{ fs_group }} +{% endif %} containers: - name: guidellm image: {{ image }} @@ -26,6 +30,14 @@ spec: env: - name: USER value: guidellm +{% if hf_token_secret %} + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: {{ hf_token_secret }} + key: HF_TOKEN + optional: true +{% endif %} volumeMounts: - name: home mountPath: /home/guidellm diff --git a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py index 55e81d035..8454f81aa 100644 --- a/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py +++ b/projects/guidellm/toolbox/run_guidellm_benchmark/utils.py @@ -146,13 +146,16 @@ def _build_multi_run_script(*, endpoint_url: str, runs: list[GuideLLMRun]) -> st return "\n".join(lines) -def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) -> dict[str, Any]: +def render_guidellm_pvc_from_parts( + *, namespace: str, name: str, pvc_size: str, owner_reference: dict[str, Any] | None = None +) -> dict[str, Any]: """Render a GuideLL-M PVC manifest from individual components. Args: namespace: Target namespace name: Name of the benchmark job and PVC pvc_size: Size of the PVC + owner_reference: Optional owner reference to set (e.g., for job ownership) Returns: PVC manifest as dict @@ -165,7 +168,13 @@ def render_guidellm_pvc_from_parts(*, namespace: str, name: str, pvc_size: str) "pvc_size": pvc_size, }, ) - return yaml.safe_load(rendered_yaml) + manifest = yaml.safe_load(rendered_yaml) + + # Add owner reference if provided + if owner_reference: + manifest["metadata"]["ownerReferences"] = [owner_reference] + + return manifest def render_guidellm_job_from_parts( @@ -175,6 +184,8 @@ def render_guidellm_job_from_parts( image: str, endpoint_url: str, guidellm_args: list[str], + hf_token_secret: str = "", + fs_group: int | None = None, ) -> dict[str, Any]: """Render a GuideLL-M job manifest from individual components. @@ -184,6 +195,10 @@ def render_guidellm_job_from_parts( image: Container image for GuideLLM endpoint_url: Gateway endpoint URL guidellm_args: Additional arguments for GuideLLM + hf_token_secret: Name of the K8s secret containing HF_TOKEN. If empty, HF_TOKEN is not injected. + fs_group: If set, adds a pod-level securityContext.fsGroup to ensure + the PVC is writable by the container. Needed on clusters where the + CSI driver provisions volumes with root-only permissions. Returns: Job manifest as dict @@ -195,6 +210,8 @@ def render_guidellm_job_from_parts( "namespace": namespace, "name": name, "image": image, + "hf_token_secret": hf_token_secret, + "fs_group": fs_group, }, ) manifest = yaml.safe_load(rendered_yaml) diff --git a/projects/guidellm/toolbox/run_smoke_request/main.py b/projects/guidellm/toolbox/run_smoke_request/main.py index 703bef5fe..eba195b84 100644 --- a/projects/guidellm/toolbox/run_smoke_request/main.py +++ b/projects/guidellm/toolbox/run_smoke_request/main.py @@ -5,7 +5,7 @@ import json import logging -from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl import always, entrypoint, execute_tasks, retry, task from projects.core.dsl.utils import write_json, write_text from projects.core.dsl.utils.k8s import ( oc, @@ -24,7 +24,7 @@ def run( namespace: str, endpoint_url: str, pod_name: str = "llm-d-smoke", - client_image: str = "curlimages/curl:8.11.1", + client_image: str = "quay.io/curl/curl:8.21.0", endpoint_path: str = "/v1/completions", request_timeout_seconds: int = 60, served_model_name: str, @@ -129,7 +129,7 @@ def create_smoke_pod(args, ctx): return f"Created smoke pod {ctx.pod_name}" -@retry(attempts=60, delay=5, backoff=1.0) +@retry(attempts=12, delay=5, backoff=1.0) @task def wait_smoke_pod_running(args, ctx): """Wait for smoke pod to be running""" @@ -224,10 +224,59 @@ def execute_smoke_request(args, ctx): return "Smoke request completed successfully" +@always +@task +def capture_smoke_pod_debug_info(args, ctx): + """Capture pod YAML and description for debugging (always runs)""" + + if not hasattr(ctx, "pod_name"): + return "No pod name available for debug capture" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + try: + # Capture pod YAML + oc( + "get", + "pod", + ctx.pod_name, + "-n", + args.namespace, + "-o", + "yaml", + check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.yaml", + ) + + # Capture pod description + oc( + "describe", + "pod", + ctx.pod_name, + "-n", + args.namespace, + check=False, + stdout_dest=artifacts_dir / f"{ctx.pod_name}.description.txt", + ) + + return f"Captured debug info for pod {ctx.pod_name}" + + except Exception as e: + # Don't fail the task if debug capture fails + logger.warning(f"Failed to capture debug info for pod {ctx.pod_name}: {e}") + return f"Failed to capture debug info: {e}" + + +@always @task def cleanup_smoke_pod(args, ctx): """Delete the smoke pod after test completion""" + if not hasattr(ctx, "pod_name"): + return "No pod name available for cleanup" + oc( "delete", "pod", @@ -235,6 +284,7 @@ def cleanup_smoke_pod(args, ctx): "-n", args.namespace, "--ignore-not-found=true", + "--wait=false", check=False, ) return f"Cleaned up smoke pod {ctx.pod_name}" diff --git a/projects/kserve/toolbox/capture_llmisvc_state/main.py b/projects/kserve/toolbox/capture_llmisvc_state/main.py index 12efa1935..2daafbddc 100644 --- a/projects/kserve/toolbox/capture_llmisvc_state/main.py +++ b/projects/kserve/toolbox/capture_llmisvc_state/main.py @@ -26,6 +26,7 @@ def setup_directories(args, context): """Create the artifacts directory""" shell.mkdir("artifacts") + shell.mkdir("artifacts/logs") return "Artifacts directory created" @@ -121,7 +122,7 @@ def capture_namespace_pods(args, context): """Capture all pods in the namespace with wide output""" shell.run( f"oc get pods -owide -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.pods.status.txt", check=False, ) return "Namespace pods status captured" @@ -132,7 +133,7 @@ def capture_namespace_services(args, context): """Capture all services in the namespace""" shell.run( f"oc get svc -n {context.target_namespace}", - stdout_dest=args.artifact_dir / "artifacts/namespace.services.status", + stdout_dest=args.artifact_dir / "artifacts/namespace.services.status.txt", check=False, ) return "Namespace services captured" @@ -173,20 +174,19 @@ def capture_pod_logs(args, context): if not pod_names or not result.stdout.strip(): return "No pods found to capture logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod logs captured for {len(pod_names)} pods" + return f"Pod logs captured for {captured_count} pods in dedicated files" @task @@ -195,26 +195,26 @@ def capture_pod_previous_logs(args, context): result = shell.run( f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].metadata.name}}"', check=False, + log_stdout=False, ) pod_names = result.stdout.strip().split() if not pod_names or not result.stdout.strip(): return "No pods found to capture previous logs" - log_file = args.artifact_dir / "artifacts/llminferenceservice.pods.previous.logs" + logs_dir = args.artifact_dir / "artifacts/logs" + captured_count = 0 - with open(log_file, "w") as handle: - for pod_name in pod_names: - handle.write(f"=== Previous logs for pod: {pod_name} ===\n") - log_result = shell.run( - f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", - check=False, - log_stdout=False, - ) - handle.write(log_result.stdout) - handle.write("\n") + for pod_name in pod_names: + log_file = logs_dir / f"{pod_name}.previous.log" + shell.run( + f"oc logs {pod_name} -n {context.target_namespace} --previous --all-containers=true", + stdout_dest=log_file, + check=False, + ) + captured_count += 1 - return f"Pod previous logs captured for {len(pod_names)} pods" + return f"Pod previous logs captured for {captured_count} pods in dedicated files" @task @@ -228,6 +228,62 @@ def capture_llminferenceservice_describe(args, context): return "LLMInferenceService describe captured" +@task +def capture_workload_overview(args, context): + """Capture deployment, replicaset, and pod overview for debugging""" + + workload_overview_path = args.artifact_dir / "artifacts/workload_overview.txt" + + # Capture deployment, replicaset, and pod overview with wide output + shell.run( + f'oc get deploy,rs,pod -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o wide', + stdout_dest=workload_overview_path, + check=False, + ) + + return f"Captured workload overview to {workload_overview_path}" + + +@task +def capture_workload_descriptions(args, context): + """Capture workload descriptions for deployments, replicasets, and pods in a single file""" + + descriptions_file = args.artifact_dir / "artifacts/workload_descriptions.txt" + + with open(descriptions_file, "w") as handle: + # Capture deployments descriptions + handle.write("=== DEPLOYMENTS ===\n") + deploy_result = shell.run( + f'oc describe deployments -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(deploy_result.stdout) + handle.write("\n\n") + + # Capture replicasets descriptions + handle.write("=== REPLICASETS ===\n") + rs_result = shell.run( + f'oc describe replicasets -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(rs_result.stdout) + handle.write("\n\n") + + # Capture pods descriptions + handle.write("=== PODS ===\n") + pods_result = shell.run( + f'oc describe pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace}', + log_stdout=False, + check=False, + ) + handle.write(pods_result.stdout) + handle.write("\n") + + return f"Captured workload descriptions to {descriptions_file}" + + @task def capture_pods_describe(args, context): """Capture describe output for related pods""" @@ -256,5 +312,126 @@ def capture_pods_describe(args, context): return f"Pod describe output captured for {len(pod_names)} pods" +@task +def determine_used_nodes(args, context): + """Determine which nodes are used by LLMInferenceService pods""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o jsonpath="{{.items[*].spec.nodeName}}"', + check=False, + log_stdout=False, + ) + + node_names = list(set(result.stdout.strip().split())) # Remove duplicates + if not node_names or not result.stdout.strip(): + context.used_nodes = [] + return "No nodes found for LLMInferenceService pods" + + context.used_nodes = [node for node in node_names if node] # Filter out empty strings + return f"Found {len(context.used_nodes)} nodes used by LLMInferenceService pods: {', '.join(context.used_nodes)}" + + +@task +def capture_pod_node_mapping(args, context): + """Capture pod->node mapping as YAML""" + result = shell.run( + f'oc get pods -l "app.kubernetes.io/name={args.llmisvc_name}" -n {context.target_namespace} -o custom-columns=POD:.metadata.name,NODE:.spec.nodeName --no-headers', + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return "No pod->node mapping found" + + mapping_file = args.artifact_dir / "artifacts/pod_node_mapping.yaml" + pod_node_mapping = {} + + for line in result.stdout.strip().split("\n"): + if line.strip(): + parts = line.split() + if len(parts) >= 2: + pod_name = parts[0] + node_name = parts[1] if parts[1] != "" else None + pod_node_mapping[pod_name] = node_name + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"pod_node_mapping": pod_node_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Pod->node mapping captured to {mapping_file} ({len(pod_node_mapping)} pods)" + + +@task +def capture_node_gpu_mapping(args, context): + """Capture node->GPU type mapping as YAML""" + if not context.used_nodes: + return "No nodes to capture GPU mapping for" + + node_gpu_mapping = {} + + for node_name in context.used_nodes: + # Try nvidia.com/gpu.product first + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.nvidia\\.com/gpu\\.product}}"', + check=False, + log_stdout=False, + ) + gpu_type = result.stdout.strip() + + # If not found, try gpu.nvidia.com/class as fallback + if not gpu_type: + result = shell.run( + f'oc get node {node_name} -o jsonpath="{{.metadata.labels.gpu\\.nvidia\\.com/class}}"', + check=False, + log_stdout=False, + ) + gpu_class = result.stdout.strip() + if gpu_class: + gpu_type = f"NVIDIA-{gpu_class}" + else: + gpu_type = "unknown" + + node_gpu_mapping[node_name] = gpu_type + + mapping_file = args.artifact_dir / "artifacts/node_gpu_mapping.yaml" + + import yaml + + with open(mapping_file, "w") as f: + yaml.safe_dump( + {"node_gpu_mapping": node_gpu_mapping, "capture_timestamp": context.capture_timestamp}, + f, + default_flow_style=False, + ) + + return f"Node->GPU mapping captured to {mapping_file} ({len(node_gpu_mapping)} nodes)" + + +@task +def capture_used_nodes_yaml(args, context): + """Capture YAML definitions for nodes used by LLMInferenceService pods""" + if not context.used_nodes: + return "No nodes to capture YAML for" + + nodes_dir = args.artifact_dir / "artifacts/nodes" + shell.mkdir("artifacts/nodes") + + captured_count = 0 + for node_name in context.used_nodes: + node_file = nodes_dir / f"{node_name}.yaml" + shell.run( + f"oc get node {node_name} -oyaml", + stdout_dest=node_file, + check=False, + ) + captured_count += 1 + + return f"Captured YAML for {captured_count} nodes in {nodes_dir}" + + if __name__ == "__main__": run.main() diff --git a/projects/kserve/toolbox/deploy_llmisvc/__init__.py b/projects/kserve/toolbox/deploy_llmisvc/__init__.py index e69de29bb..68f67b1fb 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/__init__.py +++ b/projects/kserve/toolbox/deploy_llmisvc/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from projects.kserve.toolbox.deploy_llmisvc.main import try_resolve_endpoint_url + +__all__ = ["try_resolve_endpoint_url"] diff --git a/projects/kserve/toolbox/deploy_llmisvc/main.py b/projects/kserve/toolbox/deploy_llmisvc/main.py index 5b27231bf..566ac3660 100644 --- a/projects/kserve/toolbox/deploy_llmisvc/main.py +++ b/projects/kserve/toolbox/deploy_llmisvc/main.py @@ -2,12 +2,21 @@ from __future__ import annotations +import logging from pathlib import Path import yaml -from projects.core.dsl import always, entrypoint, execute_tasks, on_failure, retry, task -from projects.core.dsl.utils import write_text +from projects.core.dsl import ( + EarlyReturn, + always, + entrypoint, + execute_tasks, + on_failure, + retry, + task, +) +from projects.core.dsl.utils import slugify_identifier, write_text from projects.core.dsl.utils.k8s import ( oc, oc_apply, @@ -16,6 +25,8 @@ from .on_failure_helpers import on_wait_pods_appear_failure +logger = logging.getLogger(__name__) + def load_yaml(path: Path): with path.open(encoding="utf-8") as handle: @@ -27,7 +38,9 @@ def run( *, namespace: str, inference_service_manifest_path: str, - gateway_status_address_name: str = "gateway-external", + gateway_status_address_name: str | None = "gateway-external", + dry_run: bool = False, + wait_pods_scheduled: bool = False, ) -> str: """ Deploy an LLMInferenceService and wait for its endpoint. @@ -36,42 +49,47 @@ def run( namespace: Namespace used by llm_d inference_service_manifest_path: Path to the InferenceService YAML manifest file gateway_status_address_name: Gateway status address name for endpoint resolution + dry_run: If True, only prepare the manifest without deploying + wait_pods_scheduled: If True, wait for all pods to be scheduled before checking service readiness """ - # Load manifest to extract the service name - manifest = load_yaml(Path(inference_service_manifest_path)) - inference_service_name = manifest["metadata"]["name"] - - # Pass only the required arguments to avoid including manifest content in logs - task_args = { - "namespace": namespace, - "inference_service_manifest_path": inference_service_manifest_path, - "inference_service_name": inference_service_name, - "gateway_status_address_name": gateway_status_address_name, - } - context = execute_tasks(task_args) + ctx = execute_tasks(locals()) + + if dry_run: + return ctx.src_manifest_path # Ensure endpoint_url is available - endpoint_url = getattr(context, "endpoint_url", None) + endpoint_url = getattr(ctx, "endpoint_url", None) if not endpoint_url: raise RuntimeError("Failed to resolve gateway endpoint URL after deployment") - # Log service description for visibility - service_description = getattr(context, "service_description", None) - if service_description: - print(f"Deployed: {service_description}") - return endpoint_url @task def copy_manifest_to_src(args, ctx): - """Copy inference service manifest to src directory for inspection and use""" + """Copy inference service manifest to src directory and extract service name""" import shutil # Get the original manifest path original_path = Path(args.inference_service_manifest_path) + # Load manifest to extract the service name + manifest = load_yaml(original_path) + ctx.inference_service_name = manifest["metadata"]["name"] + + # Validate that the service name is Kubernetes compliant + normalized_name = slugify_identifier(ctx.inference_service_name) + if normalized_name != ctx.inference_service_name: + raise ValueError( + f"LLMInferenceService name '{ctx.inference_service_name}' is not Kubernetes compliant. " + f"Expected: '{normalized_name}'. " + f"Names must be lowercase, contain only letters, numbers, and hyphens, " + f"and be 63 characters or less." + ) + + ctx.selector = f"app.kubernetes.io/name={ctx.inference_service_name}" + # Ensure the src directory exists src_dir = args.artifact_dir / "src" src_dir.mkdir(parents=True, exist_ok=True) @@ -83,14 +101,22 @@ def copy_manifest_to_src(args, ctx): # Store the src path in context for other tasks to use ctx.src_manifest_path = str(src_path) - return f"Copied manifest from {original_path} to {src_path}" + return f"Copied manifest from {original_path} to {src_path} (service: {ctx.inference_service_name})" + + +@task +def check_dry_run(args, ctx): + """Check if dry-run mode is enabled and return early if so""" + if args.dry_run: + return EarlyReturn(f"Dry-run completed: Prepared manifest for {ctx.inference_service_name}") + return "Proceeding with full deployment" @task def delete_existing_service(args, ctx): """Delete existing LLMInferenceService""" - name = args.inference_service_name + name = ctx.inference_service_name oc( "delete", "llminferenceservice", @@ -100,8 +126,7 @@ def delete_existing_service(args, ctx): "--ignore-not-found=true", check=False, ) - ctx.service_name = name - ctx.selector = f"app.kubernetes.io/name={name}" + return f"Deleted existing LLMInferenceService {name}" @@ -110,13 +135,21 @@ def delete_existing_service(args, ctx): def wait_old_pods_gone(args, ctx): """Wait for old llm-d pods to disappear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + result = oc( + "get", + "pods", + "-n", + args.namespace, + "-l", + ctx.selector, + "--ignore-not-found=true", + "--no-headers", + check=False, + ) - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) - if not pods or not pods.get("items"): - return f"Old pods gone for {service_name}" + # Check if output is empty (no pods found) + if not result.stdout.strip(): + return f"Old pods gone for {ctx.inference_service_name}" return False # Retry @@ -130,7 +163,7 @@ def apply_inference_service(args, ctx): # Load and apply the manifest from src manifest = load_yaml(Path(src_manifest_path)) oc_apply(src_manifest_path, manifest) - return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.service_name}" + return f"Applied LLMInferenceService manifest from {src_manifest_path} for {ctx.inference_service_name}" @on_failure(on_wait_pods_appear_failure) @@ -139,113 +172,26 @@ def apply_inference_service(args, ctx): def wait_pods_appear(args, ctx): """Wait for llm-d pods to appear""" - # Use service name and selector from context (if set) or create from args - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + # Use plain text output for better readability in logs + result = oc( + "get", "pods", "-n", args.namespace, "-l", ctx.selector, check=False, log_stdout=True + ) - pods = oc_get_json("pods", namespace=args.namespace, selector=selector, ignore_not_found=True) - if pods and pods.get("items"): - return f"Pods appeared for {service_name}" + # Check if pods appeared (successful command with actual pod output) + if ( + result.returncode == 0 + and result.stdout.strip() + and "No resources found" not in result.stdout + ): + return f"Pods appeared for {ctx.inference_service_name}" return False # Retry -@always -@task -def capture_llmisv_description(args, ctx): - """Capture LLMISV description with events and status for failure analysis""" - - try: - # Ensure artifacts directory exists - artifacts_dir = args.artifact_dir / "artifacts" - artifacts_dir.mkdir(parents=True, exist_ok=True) - - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) - - # Capture LLMISV description - result = oc( - "describe", - "llminferenceservice", - service_name, - "-n", - args.namespace, - log_stdout=False, - check=False, - ) - - llmisv_desc_path = artifacts_dir / "llmisv_description.txt" - with open(llmisv_desc_path, "w", encoding="utf-8") as f: - f.write(result.stdout) - - return f"Captured LLMISV description to {llmisv_desc_path}" - - except Exception as e: - return f"Failed to capture LLMISV description: {e}" - - -@always -@task -def capture_replicaset_description(args, ctx): - """Capture ReplicaSet description for pod creation failure analysis""" - - try: - # Ensure artifacts directory exists - artifacts_dir = args.artifact_dir / "artifacts" - artifacts_dir.mkdir(parents=True, exist_ok=True) - - # Use service name from args (available) or context (if set) - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") - - # Get replicasets for the service - rs_result = oc( - "get", - "replicaset", - "-l", - selector, - "-n", - args.namespace, - "-o", - "name", - log_stdout=False, - check=False, - ) - - replicaset_descriptions = [] - - if rs_result.stdout.strip(): - # Describe each replicaset - for rs_name in rs_result.stdout.strip().split("\n"): - if rs_name.strip(): - rs_desc_result = oc( - "describe", - rs_name.strip(), - "-n", - args.namespace, - log_stdout=False, - check=False, - ) - replicaset_descriptions.append(rs_desc_result.stdout) - - # Save all replicaset descriptions - rs_desc_path = artifacts_dir / "replicaset_description.txt" - with open(rs_desc_path, "w", encoding="utf-8") as f: - if replicaset_descriptions: - f.write("\n".join(replicaset_descriptions)) - else: - f.write("No replicasets found for the service") - - return f"Captured ReplicaSet description to {rs_desc_path}" - - except Exception as e: - return f"Failed to capture ReplicaSet description: {e}" - - @task def query_service_status(args, ctx): """Query the status of the LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query only the Ready condition status result = oc( @@ -272,7 +218,7 @@ def query_service_status(args, ctx): def query_service_message(args, ctx): """Query detailed message from LLMInferenceService""" - service_name = getattr(ctx, "service_name", args.inference_service_name) + service_name = ctx.inference_service_name # Query the Ready condition details result = oc( @@ -304,13 +250,69 @@ def query_service_message(args, ctx): return "No Ready condition found in status" -@retry(attempts=90, delay=10, backoff=1.0) +@retry(attempts=999999, delay=30, backoff=1.0) +@task +def wait_pods_scheduled(args, ctx): + """Wait for all pods to be scheduled (optional task)""" + + # Check if this task is enabled + if not args.wait_pods_scheduled: + return "Pod scheduling wait disabled by parameter" + + service_name = ctx.inference_service_name + + # First, check if the LLMInferenceService exists + llmisvc_result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + check=False, + ) + + if llmisvc_result.returncode != 0: + return False, f"LLMInferenceService {service_name} does not exist yet" + + # Get pod status using plain text output + result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "--no-headers", + check=False, + ) + + # Check for nonzero return code from pod query + if result.returncode != 0: + return False, f"Failed to query pods for service {service_name}" + + # Check if pod query result indicates no resources found + if "No resources found" in result.stdout: + return False, "No pods found for the service yet" + + if not result.stdout.strip(): + return False, "No pods found for the service yet" + + # Keep waiting if any pod is Pending or SchedulingGated + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + if "SchedulingGated" in result.stdout: + return False, "Waiting for pods to exit SchedulingGated state" + + return f"All pods for {service_name} are scheduled successfully" + + +@retry(attempts=180, delay=10, backoff=1.0) @task def wait_service_ready(args, ctx): """Wait for LLMInferenceService to be ready""" - service_name = getattr(ctx, "service_name", args.inference_service_name) - selector = getattr(ctx, "selector", f"app.kubernetes.io/name={service_name}") + service_name = ctx.inference_service_name # Query the current status and show diagnostic info result = oc( @@ -321,7 +323,6 @@ def wait_service_ready(args, ctx): args.namespace, "-o", "jsonpath={.status.conditions[?(@.type=='Ready')]}", - log_stdout=True, ) # Also show pod status for debugging @@ -329,12 +330,40 @@ def wait_service_ready(args, ctx): "get", "pods", "-l", - selector, + ctx.selector, "-n", args.namespace, - log_stdout=True, # Show pod status in logs ) + # Check for pod restarts and abort if any pods have restarted + restart_result = oc( + "get", + "pods", + "-l", + ctx.selector, + "-n", + args.namespace, + "-o", + "jsonpath={range .items[*]}{.metadata.name}:{.status.containerStatuses[*].restartCount}{'\\n'}{end}", + log_stdout=False, + ) + + if restart_result.stdout.strip(): + for line in restart_result.stdout.strip().split("\n"): + if ":" in line: + pod_name, restart_counts = line.split(":", 1) + # Check if any container has restarted + counts = restart_counts.split() + for count_str in counts: + try: + if int(count_str) > 0: + raise RuntimeError( + f"Pod {pod_name} has restarted (restart count: {count_str}). Aborting wait due to pod restart." + ) + except ValueError: + # Skip non-numeric restart counts + pass + if result.stdout.strip(): try: import json @@ -358,45 +387,483 @@ def wait_service_ready(args, ctx): return (False, f"No Ready condition found in status for {service_name}") -@retry(attempts=90, delay=10, backoff=1.0) +def try_resolve_endpoint_url( + *, namespace: str, inference_service_name: str, gateway_status_address_name: str | None +) -> str | None: + logger.info( + f"=== Resolving endpoint URL for {inference_service_name} in namespace {namespace} ===" + ) + logger.info(f"Target gateway_status_address_name: {gateway_status_address_name}") + + payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) + + # Log the entire status section for debugging + status = payload.get("status", {}) + logger.info(f"Status section keys found: {list(status.keys())}") + + # Check status.address first + status_address = status.get("address") + logger.info(f"status.address content: {status_address}") + + if status_address: + logger.info("✓ status.address exists") + if isinstance(status_address, dict): + logger.info("✓ status.address is a dict") + if status_address.get("url"): + url = status_address["url"] + logger.info(f"✓ Found URL in status.address: '{url}'") + + # When gateway_status_address_name is None, append port 8000 if needed + if gateway_status_address_name is None: + logger.info("Mode: No gateway (will append port 8000 if needed)") + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + logger.info(f"✓ Appended port 8000: '{url}'") + else: + logger.info("✓ URL already has port, using as-is") + else: + logger.info( + f"Mode: Gateway '{gateway_status_address_name}' (no port modification)" + ) + + logger.info(f"🎯 RESOLVED from status.address: '{url}'") + return url + else: + logger.info("✗ status.address has no 'url' field") + logger.info(f" Available fields: {list(status_address.keys())}") + else: + logger.info(f"✗ status.address is not a dict, type: {type(status_address)}") + else: + logger.info("✗ No status.address found") + + # Fallback to existing status.addresses logic + logger.info("--- Falling back to status.addresses lookup ---") + status_addresses = status.get("addresses", []) + logger.info(f"status.addresses content: {status_addresses}") + + if not status_addresses: + logger.info("✗ No status.addresses found - RESOLUTION FAILED") + return None + + logger.info(f"✓ Found {len(status_addresses)} address(es) in status.addresses") + + for i, address in enumerate(status_addresses): + logger.info(f"Checking address[{i}]: {address}") + + # When gateway_status_address_name is None, return the first address with a URL and append port 8000 + if gateway_status_address_name is None: + logger.info(" Mode: No gateway filter (looking for any URL)") + if address.get("url"): + url = address["url"] + logger.info(f" ✓ Found URL in address[{i}]: '{url}'") + # Append port 8000 when not using gateway if no port is already specified + if ":" not in url.split("/")[-1]: # Check if no port in the hostname part + url = f"{url}:8000" + logger.info(f" ✓ Appended port 8000: '{url}'") + else: + logger.info(" ✓ URL already has port, using as-is") + + logger.info(f"🎯 RESOLVED from status.addresses[{i}]: '{url}'") + return url + else: + logger.info(f" ✗ Address[{i}] has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") + # Otherwise, match by name + else: + address_name = address.get("name") + logger.info( + f" Mode: Gateway filter (looking for name='{gateway_status_address_name}')" + ) + logger.info(f" Address[{i}] name: '{address_name}'") + + if address_name == gateway_status_address_name: + logger.info(f" ✓ Name matches '{gateway_status_address_name}'") + if address.get("url"): + url = address["url"] + logger.info(f" ✓ Found URL: '{url}'") + logger.info(f"🎯 RESOLVED from status.addresses[{i}] by name: '{url}'") + return url + else: + logger.info(" ✗ Matching address has no 'url' field") + logger.info(f" Available fields: {list(address.keys())}") + else: + logger.info( + f" ✗ Name mismatch: '{address_name}' != '{gateway_status_address_name}'" + ) + + logger.info("❌ RESOLUTION FAILED - No suitable URL found in any address") + return None + + +@retry(attempts=30, delay=10, backoff=1.0) @task def resolve_endpoint_task(args, ctx): """Resolve the gateway endpoint URL""" endpoint_url = try_resolve_endpoint_url( namespace=args.namespace, - inference_service_name=args.inference_service_name, + inference_service_name=ctx.inference_service_name, gateway_status_address_name=args.gateway_status_address_name, ) if endpoint_url: ctx.endpoint_url = endpoint_url write_text(args.artifact_dir / "artifacts" / "endpoint.url", f"{endpoint_url}\n") return f"Endpoint resolved: {endpoint_url}" - return False # Retry + return False, "No endpoint URL available" +@always @task -def deploy_llmisvc_task(args, ctx): - """Deploy the llm_d inference service and resolve its endpoint""" +def capture_llmisv_description(args, ctx): + """Capture LLMISV description with events and status for failure analysis""" - # All work is done by the individual tasks - service_info = getattr( - ctx, "service_description", f"LLM Service '{args.inference_service_name}'" - ) - return ( - f"LLMInferenceService deployment completed: {service_info} - Endpoint: {ctx.endpoint_url}" + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" + + # Capture LLMISV description + result = oc( + "describe", + "llminferenceservice", + service_name, + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + + llmisv_desc_path = artifacts_dir / "llmisv_description.txt" + with open(llmisv_desc_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured LLMISV description to {llmisv_desc_path}" + + except Exception as e: + return f"Failed to capture LLMISV description: {e}" + + +@always +@task +def capture_replicaset_description(args, ctx): + """Capture ReplicaSet description for pod creation failure analysis""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" + + # Get replicasets for the service + rs_result = oc( + "get", + "replicaset", + "-l", + ctx.selector, + "-n", + args.namespace, + "-o", + "name", + log_stdout=False, + check=False, + ) + + replicaset_descriptions = [] + + if rs_result.stdout.strip(): + # Describe each replicaset + for rs_name in rs_result.stdout.strip().split("\n"): + if not rs_name.strip(): + continue + + rs_desc_result = oc( + "describe", + rs_name.strip(), + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + replicaset_descriptions.append(rs_desc_result.stdout) + + # Save all replicaset descriptions + rs_desc_path = artifacts_dir / "replicaset_description.txt" + with open(rs_desc_path, "w", encoding="utf-8") as f: + if replicaset_descriptions: + f.write("\n".join(replicaset_descriptions)) + else: + f.write("No replicasets found for the service") + + return f"Captured ReplicaSet description to {rs_desc_path}" + + except Exception as e: + return f"Failed to capture ReplicaSet description: {e}" + + +@always +@task +def capture_final_llmisvc_yaml(args, ctx): + """Capture the final YAML state of the LLMInferenceService""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use LLMInferenceService name from context + service_name = getattr(ctx, "inference_service_name", None) + if not service_name: + return "No service name available" + + # Capture final YAML state + result = oc( + "get", + "llminferenceservice", + service_name, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + llmisvc_yaml_path = artifacts_dir / "llmisvc_final.yaml" + + if result.returncode != 0: + # Handle the case where the LLMInferenceService is not found + if "not found" in result.stderr.lower(): + logger.warning( + f"LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write( + f"# LLMInferenceService '{service_name}' not found in namespace '{args.namespace}'\n" + ) + f.write(f"# Error: {result.stderr.strip()}\n") + return f"LLMInferenceService not found, logged error to {llmisvc_yaml_path}" + else: + # Handle other error cases + logger.error( + f"Failed to get LLMInferenceService '{service_name}': {result.stderr.strip()}" + ) + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(f"# Error getting LLMInferenceService '{service_name}'\n") + f.write(f"# Error: {result.stderr.strip()}\n") + return f"Error getting LLMInferenceService, logged error to {llmisvc_yaml_path}" + + # Success case - write the YAML content + with open(llmisvc_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured final LLMInferenceService YAML to {llmisvc_yaml_path}" + + except Exception as e: + return f"Failed to capture final LLMInferenceService YAML: {e}" + + +@always +@task +def capture_workload_overview(args, ctx): + """Capture deployment, replicaset, and pod overview for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + workload_overview_path = artifacts_dir / "workload_overview.txt" + + # Capture deployment, replicaset, and pod overview + oc( + "get", + "deploy,rs,pod", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + check=False, + stdout_dest=workload_overview_path, ) + return f"Captured workload overview to {workload_overview_path}" -def try_resolve_endpoint_url( - *, namespace: str, inference_service_name: str, gateway_status_address_name: str -) -> str | None: - payload = oc_get_json("llminferenceservice", name=inference_service_name, namespace=namespace) - for address in payload.get("status", {}).get("addresses", []): - if address.get("name") == gateway_status_address_name and address.get("url"): - return address["url"] - return None +@always +@task +def capture_pod_status(args, ctx): + """Capture pod status for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture pod status with wide output + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "wide", + log_stdout=False, + check=False, + ) + + pod_status_path = artifacts_dir / "pod_status.txt" + with open(pod_status_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod status to {pod_status_path}" + + except Exception as e: + return f"Failed to capture pod status: {e}" + + +@always +@task +def capture_pod_descriptions(args, ctx): + """Capture pod descriptions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Get pod names + pod_result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "jsonpath={.items[*].metadata.name}", + log_stdout=False, + check=False, + ) + + pod_names = pod_result.stdout.strip().split() + if not pod_names or not pod_result.stdout.strip(): + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("No pods found for the service") + return f"No pods found, wrote empty file to {pod_desc_path}" + + # Describe each pod + pod_descriptions = [] + for pod_name in pod_names: + describe_result = oc( + "describe", + "pod", + pod_name, + "-n", + args.namespace, + log_stdout=False, + check=False, + ) + pod_descriptions.append( + f"=== Description for pod: {pod_name} ===\n{describe_result.stdout}" + ) + + # Save all pod descriptions + pod_desc_path = artifacts_dir / "pod_descriptions.txt" + with open(pod_desc_path, "w", encoding="utf-8") as f: + f.write("\n\n".join(pod_descriptions)) + + return f"Captured descriptions for {len(pod_names)} pods to {pod_desc_path}" + + except Exception as e: + return f"Failed to capture pod descriptions: {e}" + + +@always +@task +def capture_pod_yaml(args, ctx): + """Capture pod YAML definitions for debugging""" + + if args.dry_run: + return "Dry-run, nothing to do" + + try: + # Ensure artifacts directory exists + artifacts_dir = args.artifact_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + + # Use selector from context + selector = getattr(ctx, "selector", None) + if not selector: + return "No selector available" + + # Capture all pod YAMLs + result = oc( + "get", + "pods", + "-l", + selector, + "-n", + args.namespace, + "-o", + "yaml", + log_stdout=False, + check=False, + ) + + pod_yaml_path = artifacts_dir / "pod_definitions.yaml" + with open(pod_yaml_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + + return f"Captured pod YAML definitions to {pod_yaml_path}" + + except Exception as e: + return f"Failed to capture pod YAML: {e}" if __name__ == "__main__": diff --git a/projects/kserve/toolbox/prepare_hf_model_cache/main.py b/projects/kserve/toolbox/prepare_hf_model_cache/main.py index 93c747a80..1e01c1737 100644 --- a/projects/kserve/toolbox/prepare_hf_model_cache/main.py +++ b/projects/kserve/toolbox/prepare_hf_model_cache/main.py @@ -11,7 +11,6 @@ slugify_identifier, truncate_k8s_name, write_json, - write_text, write_yaml, ) from projects.core.dsl.utils.k8s import ( @@ -36,7 +35,6 @@ def run( *, namespace: str, - namespace_is_managed: bool, model_key: str, model_uri: str, pvc_size: str, @@ -55,7 +53,6 @@ def run( Args: namespace: Namespace where the workload runs - namespace_is_managed: Whether namespace lifecycle is managed by llm_d model_key: Selected model key model_uri: HuggingFace model URI (hf://...) pvc_size: Size of the PVC to create @@ -113,13 +110,6 @@ def build_cache_spec(args, ctx): "oci_registry_auth_secret_key": None, } - if args.namespace_is_managed: - logger.warning( - "Model cache PVC %s lives in managed namespace %s. Namespace cleanup will remove it; cache reuse requires a stable namespace override.", - cache_spec["pvc_name"], - cache_spec["namespace"], - ) - ctx.cache_spec = cache_spec ctx.hf_secret_created = False ctx.hf_secret_name = None @@ -296,7 +286,45 @@ def create_download_job(args, ctx): return f"Download job {cache_spec['download_job_name']} created" -@retry(attempts=30, delay=30, backoff=1.0) +@retry(attempts=24, delay=5, backoff=1.0) # 24 attempts * 5 seconds = 120 seconds (2 minutes) +@task +def wait_for_pods_running(args, ctx): + """Wait for the download job pods to start running within 2 minutes""" + + if getattr(ctx, "cache_ready", False): + return "Skipping pod wait - cache already ready" + + cache_spec = ctx.cache_spec + job_name = cache_spec["download_job_name"] + namespace = cache_spec["namespace"] + + # Get pod status using plain text output + result = oc( + "get", + "pod", + "--no-headers", + "-l", + f"job-name={job_name}", + "-n", + namespace, + "-o", + "custom-columns=NAME:.metadata.name,STATUS:.status.phase", + check=False, + log_stdout=False, + ) + + if not result.stdout.strip(): + return False # No pods yet, retry + + # Keep waiting only if "Pending" appears in the output + if "Pending" in result.stdout: + return False, "Waiting for pods to exit Pending state" + + # All pods are past Pending state + return "All pods past Pending state" + + +@retry(attempts=120, delay=30, backoff=1.0) @task def wait_for_download(args, ctx): """Wait for the download job to complete""" @@ -365,7 +393,7 @@ def capture_download_artifacts(args, ctx): ) # Capture PVC YAML to file (not logs) - pvc_result = oc( + oc( "get", "persistentvolumeclaim", cache_spec["pvc_name"], @@ -376,12 +404,11 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "pvc.yaml", ) - if pvc_result.returncode == 0 and pvc_result.stdout: - write_text(artifact_dir / "pvc.yaml", pvc_result.stdout) # Capture Job YAML to file (not logs) - job_result = oc( + oc( "get", "job", cache_spec["download_job_name"], @@ -392,14 +419,13 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / "job.yaml", ) - if job_result.returncode == 0 and job_result.stdout: - write_text(artifact_dir / "job.yaml", job_result.stdout) # Capture pod logs and YAML to files (not console) for pod_name in job_pod_names(cache_spec["download_job_name"], cache_spec["namespace"]): # Get pod YAML to file - pod_result = oc( + oc( "get", "pod", pod_name, @@ -410,21 +436,19 @@ def capture_download_artifacts(args, ctx): "--ignore-not-found=true", check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.yaml", ) - if pod_result.returncode == 0 and pod_result.stdout: - write_text(artifact_dir / f"{pod_name}.yaml", pod_result.stdout) # Get pod logs to file - log_result = oc( + oc( "logs", pod_name, "-n", cache_spec["namespace"], check=False, log_stdout=False, + stdout_dest=artifact_dir / f"{pod_name}.log", ) - if log_result.returncode == 0 and log_result.stdout: - write_text(artifact_dir / f"{pod_name}.log", log_result.stdout) return f"Artifacts captured for {cache_spec['pvc_name']}" diff --git a/projects/kserve/toolbox/wait_kserve_ready/main.py b/projects/kserve/toolbox/wait_kserve_ready/main.py new file mode 100644 index 000000000..426cf2355 --- /dev/null +++ b/projects/kserve/toolbox/wait_kserve_ready/main.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import yaml + +from projects.core.dsl import entrypoint, execute_tasks, retry, task +from projects.core.dsl.utils.k8s import oc + +SERVING_CONTROL_PLANE_DEPLOYMENTS = ( + "kserve-controller-manager", + "llmisvc-controller-manager", + "model-serving-api", + "odh-model-controller", +) + + +@entrypoint +def run(*, namespace: str) -> str: + """ + Wait for the KServe serving control plane deployments to be ready. + + Args: + namespace: Namespace where the serving control plane is deployed + """ + + task_args = {"namespace": namespace} + execute_tasks(task_args) + + return f"Serving control plane is ready in {namespace}" + + +@task +def wait_for_deployments(args, ctx): + """Wait for all serving control plane deployments to be available""" + + for deployment_name in SERVING_CONTROL_PLANE_DEPLOYMENTS: + result = oc( + "wait", + "--for=condition=Available", + "--timeout=300s", + f"deployment/{deployment_name}", + "-n", + args.namespace, + check=False, + ) + if result.returncode != 0: + status_result = oc( + "get", + "deployments", + "-n", + args.namespace, + "-o", + "wide", + check=False, + ) + error_msg = ( + f"Timeout waiting for serving control plane deployment " + f"{deployment_name} in {args.namespace}" + ) + if status_result.returncode == 0: + error_msg += f"\n\nCurrent deployment status:\n{status_result.stdout}" + raise RuntimeError(error_msg) + + return "All serving control plane deployments are available" + + +@retry(attempts=40, delay=5, backoff=1.0) +@task +def probe_webhook_ready(args, ctx): + """Probe the admission webhook with a dry-run apply to confirm it is serving""" + + probe_manifest = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "LLMInferenceService", + "metadata": { + "name": "forge-webhook-probe", + "namespace": args.namespace, + }, + "spec": { + "model": { + "uri": "hf://probe/model", + "name": "probe", + }, + }, + } + + probe_dir = args.artifact_dir / "src" + probe_dir.mkdir(parents=True, exist_ok=True) + probe_path = probe_dir / "webhook-probe.yaml" + with open(probe_path, "w", encoding="utf-8") as f: + yaml.safe_dump(probe_manifest, f, sort_keys=False) + + result = oc( + "apply", + "--dry-run=server", + "-f", + str(probe_path), + "-n", + args.namespace, + check=False, + ) + + if result.returncode == 0: + return "Webhook is ready (dry-run accepted)" + + return False + + +if __name__ == "__main__": + run.main() diff --git a/projects/llm_d/CHANGELOG-orchestration.md b/projects/llm_d/CHANGELOG-orchestration.md index 5ccfc73a7..c471d93c2 100644 --- a/projects/llm_d/CHANGELOG-orchestration.md +++ b/projects/llm_d/CHANGELOG-orchestration.md @@ -1,5 +1,82 @@ # LLM-D Orchestration Changelog +## 2026-07-21 - Enhanced P/D Configuration System + +### P/D Profile Name-Based Configuration +- **FROM_NAME Parameter Extraction**: Implemented automatic parameter extraction from P/D profile names + - **Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + - **Tensor Parallelism**: Separate extraction for prefill (`p.tp4`) vs decode (`d.tp4` or `tp4`) containers + - **Pod Replicas**: Separate extraction for prefill (`p.x8`) vs decode (`d.x2` or `x2`) pods + - **Strict Parsing**: Removed fallback logic for precise parameter extraction + +### Template-Based Deployment Profiles +- **Template System**: Added `template__pd` with `FROM_NAME` placeholders for dynamic configuration + - **Auto-Resolution**: Profile names automatically populate `tensor_parallelism`, `prefill_pods`, `decode_pods` + - **Scheduler Integration**: Template profiles support simplified scheduler configuration (`scheduler_manifest: {}`) + +### Global P/D Configuration Architecture +- **Centralized P/D Config**: Moved P/D-specific settings to global `deployments.pd` configuration + - **VLLM Extra Args**: Global P/D VLLM arguments (e.g., `--block-size`, `--kv-transfer-config`) + - **Environment Variables**: Global P/D environment variables (e.g., `VLLM_NIXL_SIDE_CHANNEL_HOST`) + - **Resource Requirements**: Global P/D resource specifications (e.g., `rdma/ib: '1'`) + +### Enhanced VLLM Arguments System +- **Configuration Rename**: Changed `vllm_args` → `vllm_extra_args` for clarity +- **Conditional Behavior**: Different VLLM arg handling based on `use_kserve_defaults` flag + - **KServe Mode**: Only extra args when `use_kserve_defaults: true` + - **Standard Mode**: Extra args + automatic `--tensor-parallel-size` when `use_kserve_defaults: false` +- **P/D Integration**: P/D deployments receive both profile args and global P/D args with correct tensor parallelism + +### Resource Allocation Improvements +- **Per-Pod-Type Configuration**: Different resource allocation for prefill vs decode pods + - **Prefill Pods**: Use prefill tensor parallelism (extracted from `p.tp{N}`) + - **Decode Pods**: Use main tensor parallelism (extracted from `d.tp{N}` or `tp{N}`) +- **Resource Merging**: Automatic merging of base resources with P/D-specific resources + +### Testing Infrastructure Enhancements +- **Annotation-Based Profile Detection**: Added `forge.openshift.io/deployment-profile` annotation for reliable profile identification + - **Replaced Heuristics**: Eliminated complex directory name parsing in test system + - **Reliable Testing**: Tests now read profile names directly from manifest annotations +- **Namespace Configuration**: Fixed namespace override behavior for consistent testing + +### Environment and Artifact Management +- **Artifact Directory Naming**: Enhanced artifact directory creation with conflict resolution + - **Base Format**: `forge_YYYYMMDD-HHMM` without seconds when possible + - **Conflict Resolution**: Add seconds (`forge_YYYYMMDD-HHMM-SS`) when directory exists +- **Symlink Management**: Automatic `/tmp/forge_last` symlink to latest artifact directory + +### Files Modified +- `projects/llm_d/orchestration/runtime_config.py` - FROM_NAME extraction system, template resolution, parameter parsing +- `projects/llm_d/orchestration/render_inference_service.py` - P/D rendering, VLLM args system, resource allocation, annotations +- `projects/llm_d/orchestration/config.d/deployments.yaml` - Template profiles, global P/D configuration +- `projects/llm_d/orchestration/config.d/runtime.yaml` - Namespace override, kueue configuration updates +- `projects/llm_d/tests/test_deployment_profiles.py` - Annotation-based profile detection, namespace override +- `projects/core/library/env.py` - Artifact directory conflict resolution, symlink management +- `projects/llm_d/docs/PD_TESTING.md` - Updated documentation for new P/D configuration system + +### Benefits +- **Flexible Configuration**: Profile names encode all P/D parameters for easy testing +- **Template Reuse**: Single template profile supports multiple P/D configurations +- **Resource Optimization**: Different GPU allocation for prefill vs decode workloads +- **Simplified Testing**: Annotation-based profile detection eliminates complex heuristics +- **Clean Architecture**: Global P/D configuration reduces duplication across profiles + +## 2026-07-16 - Preset Configuration Support + +### Configuration Management +- **Preset CLI Option**: Added `--preset` flag to CI entrypoint for preset configuration before execution + - **Multiple Presets**: Supports multiple preset values via repeated `--preset` flags + - **Early Initialization**: Sets preset configuration during environment initialization, before config loading + - **Variables Override Integration**: Uses framework helper to generate proper override files + +### Files Modified +- `projects/llm_d/orchestration/ci.py` - Added `--presets` CLI option and initialization logic + +### Benefits +- **Flexible Testing**: Enables running same test suite with different preset configurations +- **Command-Line Control**: Provides runtime preset selection without modifying configuration files +- **Framework Integration**: Leverages core configuration override system for consistent behavior + ## 2026-06-26 - Orchestration & Cleanup Improvements ### New Features diff --git a/projects/llm_d/docs/PD_TESTING.md b/projects/llm_d/docs/PD_TESTING.md new file mode 100644 index 000000000..aeb2c3d8d --- /dev/null +++ b/projects/llm_d/docs/PD_TESTING.md @@ -0,0 +1,381 @@ +# P/D (Prefill/Decode) Testing in llm-d + +This document describes the P/D testing capabilities in the llm-d project, which allows testing various Prefill/Decode configurations with flexible pod counts, tensor parallelism, and scheduler configurations using profile name-based parameter extraction. + +## Overview + +P/D testing enables you to: +- Configure the number of prefill and decode pods via profile names +- Set different tensor parallelism for prefill vs decode pods +- Test various scheduler configurations +- Run matrix tests across multiple P/D configurations +- Use template-based profiles with automatic parameter extraction + +## P/D Configuration Architecture + +### Profile Name-Based Configuration + +P/D deployments use profile names to encode configuration parameters: + +**Pattern**: `pd-d.x{decode_pods}-p.tp{prefill_tp}-d.tp{decode_tp}-p.x{prefill_pods}` + +**Examples**: +- `pd-d.x2-p.tp1-d.tp4-p.x8` - 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-d.x1-p.tp2-d.tp8-p.x4` - 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +### Global P/D Configuration + +P/D-specific settings are centralized in `deployments.yaml`: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Template-Based Profiles + +Template profiles use `FROM_NAME` placeholders for automatic parameter extraction: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME # extracted from profile name + pd_config: + prefill_pods: FROM_NAME # extracted from p.x{N} pattern + decode_pods: FROM_NAME # extracted from d.x{N} pattern + scheduler_config: "high-throughput" +``` + +## P/D Profile Examples + +### Small Scale P/D +```yaml +# Profile: pd-d.x1-p.tp1-d.tp2-p.x2 +# 1 decode pod with 2 GPUs, 2 prefill pods with 1 GPU each +deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 +``` + +### Medium Scale P/D +```yaml +# Profile: pd-d.x2-p.tp2-d.tp4-p.x4 +# 2 decode pods with 4 GPUs each, 4 prefill pods with 2 GPUs each +deployment_profile: pd-d.x2-p.tp2-d.tp4-p.x4 +``` + +### Large Scale P/D +```yaml +# Profile: pd-d.x4-p.tp4-d.tp8-p.x8 +# 4 decode pods with 8 GPUs each, 8 prefill pods with 4 GPUs each +deployment_profile: pd-d.x4-p.tp4-d.tp8-p.x8 +``` + +## P/D Testing with Presets + +P/D testing integrates with the existing preset system. Use presets that include P/D profile configurations: + +### Custom P/D Testing +```bash +llm_d ci --preset your-pd-preset +``` + +Where your preset contains P/D profile configurations in `presets.yaml`: + +```yaml +your-pd-preset: + runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 + benchmark_key: short +``` + +## Custom P/D Testing via PR Comments + +You can override P/D configurations using PR comment directives: + +### Test Specific P/D Configurations +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +/var runtime.model_name: meta-llama/Llama-3.1-8B-Instruct +/var runtime.benchmark_key: short +``` + +### Test Different P/D Scales +``` +/var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp4-p.x1, pd-d.x4-p.tp4-d.tp8-p.x8] +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.benchmark_key: [short, concurrent-1k-1k] +``` + +### Mix P/D and Traditional Profiles +``` +/var runtime.deployment_profile: [pd-d.x2-p.tp1-d.tp4-p.x8, approximate-prefix-cache, precise-prefix-cache] +/var runtime.model_name: Qwen/Qwen3-0.6B +``` + +## Parameter Extraction Rules + +The system extracts parameters from P/D profile names using these patterns: + +### Tensor Parallelism +- **Main container (decode)**: `d.tp{N}` or `tp{N}` → N GPUs per decode pod +- **Prefill container**: `p.tp{N}` → N GPUs per prefill pod + +### Pod Replicas +- **Main container (decode)**: `d.x{N}` or `x{N}` → N decode pods +- **Prefill container**: `p.x{N}` → N prefill pods + +### Examples +- `pd-d.x2-p.tp1-d.tp4-p.x8`: 2 decode pods (4 GPUs each), 8 prefill pods (1 GPU each) +- `pd-tp8-x1-p.tp2-p.x4`: 1 decode pod (8 GPUs), 4 prefill pods (2 GPUs each) + +## Runtime Configuration Functions + +New runtime configuration functions are available for P/D testing: + +```python +from projects.llm_d.orchestration import runtime_config + +# Get P/D configuration +pd_config = runtime_config.get_pd_config() +# Returns: {'prefill_pods': 2, 'decode_pods': 4, 'scheduler_config': 'high-throughput'} + +# Get individual P/D parameters +prefill_pods = runtime_config.get_prefill_pod_count() # Returns: 2 +decode_pods = runtime_config.get_decode_pod_count() # Returns: 4 +scheduler_config = runtime_config.get_scheduler_config() # Returns: 'high-throughput' + +# Check if current deployment is P/D +is_pd = runtime_config.is_pd_deployment() # Returns: True for P/D profiles +``` + +## Test Output and Artifacts + +P/D tests generate the same artifacts as standard llm-d tests, with additional P/D-specific metadata: + +### Test Labels +```yaml +model_name: meta-llama/Llama-3.1-8B-Instruct +deployment_profile: pd-d.x2-p.tp1-d.tp4-p.x8 +guidellm_loadshape: short +pd_prefill_pods: 8 # Extracted from p.x8 +pd_decode_pods: 2 # Extracted from d.x2 +pd_prefill_tensor_parallelism: 1 # Extracted from p.tp1 +pd_decode_tensor_parallelism: 4 # Extracted from d.tp4 +pd_scheduler_config: high-throughput +``` + +### Artifact Directory Structure +``` +ARTIFACT_DIR/ +├── llmd_test/ +│ ├── __test_labels__.yaml +│ ├── artifacts/ +│ │ ├── llminferenceservice.yaml +│ │ ├── endpoint.url +│ │ └── results/ +│ │ └── benchmark_short/ +├── prepare_llmd_run_pd_d_x2_p_tp1_d_tp4_p_x8_meta_llama_instruct/ +└── ... +``` + +### LLMISVC Manifest Annotations +P/D deployments include profile metadata in annotations: + +```yaml +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d.x2-p.tp1-d.tp4-p.x8 + namespace: forge-llm-d + annotations: + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 +spec: + replicas: 2 # decode pods + prefill: + replicas: 8 # prefill pods + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' # prefill tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + template: # decode template + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' # decode tensor parallelism + rdma/ib: '1' # P/D resources + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' +``` + +## Implementation Details + +### Template Profile Configuration +P/D template profiles are defined in `orchestration/config.d/deployments.yaml`: + +```yaml +template__pd: + scheduler_manifest: {} # simplified scheduler + tensor_parallelism: FROM_NAME + pd_config: + prefill_pods: FROM_NAME + decode_pods: FROM_NAME + scheduler_config: "high-throughput" +``` + +### Global P/D Configuration +P/D-specific VLLM args, environment variables, and resources: + +```yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' +``` + +### Parameter Extraction +The `_extract_value_from_profile_name()` function extracts values using regex patterns: + +```python +# For tensor_parallelism (main container) +match = re.search(r"d\.tp(\d+)", profile_name) # d.tp4 → 4 +if not match: + match = re.search(r"(? my-pd-preset.yaml << EOF +runtime: + model_name: Qwen/Qwen3-0.6B + deployment_profile: pd-d.x1-p.tp1-d.tp2-p.x2 + benchmark_key: short +EOF + +llm_d ci --preset my-pd-preset +``` + +### Matrix P/D Testing +```bash +# Use PR comment for matrix testing: +# /var runtime.deployment_profile: [pd-d.x1-p.tp1-d.tp2-p.x2, pd-d.x2-p.tp2-d.tp4-p.x4] +# /var runtime.model_name: [Qwen/Qwen3-0.6B, meta-llama/Llama-3.1-8B-Instruct] +# /var runtime.benchmark_key: short + +llm_d ci # Will use PR directive overrides +``` + +### Custom P/D with Different GPU Allocation +```bash +# Test asymmetric GPU allocation: many small prefill pods, few large decode pods +# Profile: pd-d.x1-p.tp1-d.tp8-p.x16 +# 1 decode pod with 8 GPUs, 16 prefill pods with 1 GPU each + +cat > asymmetric-pd.yaml << EOF +runtime: + model_name: meta-llama/Llama-3.1-8B-Instruct + deployment_profile: pd-d.x1-p.tp1-d.tp8-p.x16 + benchmark_key: concurrent-1k-1k +EOF + +llm_d ci --preset asymmetric-pd +``` + +### Validating P/D Profile Names +```bash +# Test profile name parsing +python3 -c " +from projects.llm_d.orchestration.runtime_config import _extract_value_from_profile_name + +profile = 'pd-d.x2-p.tp1-d.tp4-p.x8' +print(f'Decode pods: {_extract_value_from_profile_name(profile, \"decode_pods\")}') +print(f'Prefill pods: {_extract_value_from_profile_name(profile, \"prefill_pods\")}') +print(f'Decode TP: {_extract_value_from_profile_name(profile, \"tensor_parallelism\")}') +print(f'Prefill TP: {_extract_value_from_profile_name(profile, \"prefill_tensor_parallelism\")}') +" +``` + +This P/D testing system provides flexible, template-based configuration with automatic parameter extraction from profile names, enabling sophisticated testing of Prefill/Decode configurations at scale. diff --git a/projects/llm_d/docs/benchmark-isolation.md b/projects/llm_d/docs/benchmark-isolation.md new file mode 100644 index 000000000..3fae5d2b0 --- /dev/null +++ b/projects/llm_d/docs/benchmark-isolation.md @@ -0,0 +1,111 @@ +# Benchmark Isolation: Per-Workload Deployment + +## Why each benchmark needs a fresh deployment + +The llm_d test harness deploys an LLMInferenceService (LLMISVC) for each +benchmark workload rather than sharing a single deployment across workloads. +Three factors drive this design: + +### 1. `--max-model-len` affects performance + +The vLLM serving engine's `--max-model-len` parameter controls the maximum +sequence length and directly impacts memory allocation, scheduling behavior, +and throughput. Different workloads have different token distributions: + +- `short` uses `prompt_tokens=256, output_tokens=128` +- `heavy-heterogeneous` uses prompts up to 30,000 tokens + +Running `heavy-heterogeneous` against a server configured with +`--max-model-len=4096` would truncate or reject requests, while running +`short` against `--max-model-len=32768` wastes GPU memory on unused KV-cache +blocks. Workloads can provide `deployment_overrides` to set the right +`--max-model-len` for their token distribution. + +### 2. KV-cache pollution across workloads + +When multiple benchmarks share a deployment, residual KV-cache entries from +the first workload can affect the performance measurements of subsequent +workloads. The current mitigation relies on guidellm's prefixed prompts to +generate unique cache keys per benchmark. This works for standard load +generation but is not a guarantee of isolation. + +### 3. Future trace-replay workloads + +Trace-replay workloads (replaying production request traces) will require +starting the vLLM server with `VLLM_DEV_MODE=1` so that the KV-cache can +be explicitly cleared between runs. This is a deployment-time environment +variable, not a runtime toggle, reinforcing the need for per-workload +deployment isolation. + +## Architecture: 3D RunSpec matrix + +The test matrix is a Cartesian product of three dimensions: + +``` +model_name x deployment_profile x benchmark_key +``` + +Each combination (a `RunSpec`) gets: +1. Its own Kubernetes namespace +2. A fresh LLMISVC deployment with the profile's vLLM args +3. A smoke test to verify the deployment is healthy +4. A single benchmark run +5. Full teardown (LLMISVC, benchmark job, PVC, namespace resources) + +### Example + +``` +/var runtime.model_name: Qwen/Qwen3-0.6B +/var runtime.deployment_profile: [approximate-prefix-cache, distributed-default] +/var runtime.benchmark_key: [short, multi-turn] +``` + +Produces 1 x 2 x 2 = 4 isolated test runs, each with its own namespace: + +| RunSpec | Model | Profile | Benchmark | +|---------|-------|---------|-----------| +| 1 | Qwen3-0.6B | approximate-prefix-cache | short | +| 2 | Qwen3-0.6B | approximate-prefix-cache | multi-turn | +| 3 | Qwen3-0.6B | distributed-default | short | +| 4 | Qwen3-0.6B | distributed-default | multi-turn | + +## Backward compatibility + +| Scenario | `benchmark_key` value | Matrix | Behavior | +|----------|-----------------------|--------|----------| +| Smoke-only | `null` | M x P | No benchmark, just smoke test | +| Single benchmark | `"short"` | M x P x 1 | One benchmark per deploy (same as before) | +| Multiple benchmarks | `["a", "b"]` | M x P x 2 | Each gets its own deployment | + +When `benchmark_key` is null or a single scalar, the matrix degrades to the +previous 2D behavior with no overhead. + +## Per-workload deployment overrides + +Workloads can override deployment profile settings via `deployment_overrides` +in `config.d/workloads.yaml`. This dict is deep-merged on top of the resolved +deployment profile when building the LLMISVC manifest. Lists (like `vllm_args`) +are **replaced entirely**, not concatenated — this avoids duplicate or +conflicting flags when the override supplies a different set of arguments. + +```yaml +benchmarks: + heavy-heterogeneous: + args: + backend_type: openai_http + rate_type: concurrent + rate: [300, 200, 100, 50, 1] + data: prompt_tokens=8000,... + deployment_overrides: + vllm_args: + - --max-model-len=32768 + + trace-replay: + args: + backend_type: openai_http + ... + deployment_overrides: + env: + - name: VLLM_DEV_MODE + value: "1" +``` diff --git a/projects/llm_d/orchestration/ci.py b/projects/llm_d/orchestration/ci.py index d08376451..cb0adc76d 100755 --- a/projects/llm_d/orchestration/ci.py +++ b/projects/llm_d/orchestration/ci.py @@ -10,6 +10,7 @@ import click +from projects.caliper.orchestration.postprocess_outcome import TestPhaseOutcome from projects.core.agentic.config_review import trigger_config_review_for_ci from projects.core.agentic.on_failure import agent_review_on_failure from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint @@ -21,35 +22,80 @@ caliper_export_list_optional_vaults, caliper_export_list_vaults, ) +from projects.core.library.postprocess import run_postprocess_after_test from projects.core.library.replot import caliper_replot_entrypoint from projects.llm_d.orchestration.cleanup_phase import run as cleanup_toolbox_run from projects.llm_d.orchestration.preflight_phase import run as preflight_toolbox_run from projects.llm_d.orchestration.prepare_sequence import run_prepare_sequence -from projects.llm_d.orchestration.test_phase import run as test_toolbox_run logger = logging.getLogger(__name__) +RHOAI_CUSTOM_CATALOG_VAULTS = [ + "psap-rhoai-rc", + "psap-forge-staging-image-pull", +] -def init(): +def init(presets=None): """Initialize LLM-D orchestration environment""" env.init() run.init() + + # Set presets configuration if provided + if presets: + config.write_variables_override(presets=presets) + config.init(Path(__file__).parent) +def list_vaults() -> list[str]: + """List all vaults (includes both mandatory and optional).""" + all_vaults = vault.phase_vault_list_all() + if config.project.get_config("platform.rhoai.custom_catalog.enabled", False): + return [*all_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] + + return all_vaults + + +def init_vaults_for_phase(phase: str) -> None: + mandatory_vaults = [ + *config.project.get_config("vaults.all", []), + *config.project.get_config(f"vaults.{phase}", []), + ] + optional_vaults = [ + *config.project.get_config("vaults.all-optional", []), + *config.project.get_config(f"vaults.{phase}-optional", []), + ] + + if phase == "prepare" and config.project.get_config( + "platform.rhoai.custom_catalog.enabled", False + ): + mandatory_vaults = [*mandatory_vaults, *RHOAI_CUSTOM_CATALOG_VAULTS] + + vault.init(mandatory_vaults=mandatory_vaults, optional_vaults=optional_vaults) + + @click.group(cls=ci_lib.HelpfulGroup) +@click.option("--preset", multiple=True, help="Set preset configuration before starting") @click.pass_context @ci_lib.safe_ci_function -def main(ctx): +def main(ctx, preset): """LLM-D Project CI Operations for FORGE.""" ctx.ensure_object(types.SimpleNamespace) - init() + + presets_list = list(preset) + if presets_list and "," in presets_list[0]: + lst = presets_list.pop(0) + presets_list = lst.split(",") + presets_list + + logger.info(f"Setting preset configuration from CLI: {presets_list}") + + init(presets_list) if ctx.invoked_subcommand == "resolve-fournos-config": logger.info("No need to initialize the vaults for the resolve step") return - vault.phase_vault_init(ctx.invoked_subcommand) + init_vaults_for_phase(ctx.invoked_subcommand) @main.command() @@ -76,10 +122,38 @@ def preflight(ctx) -> int: @agent_review_on_failure def test(ctx) -> int: """Test phase - Execute the main testing logic.""" + # Trigger config review analysis asynchronously (don't block test execution) trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) - return test_toolbox_run() + # Run all tests first + from projects.llm_d.orchestration.test_phase import run_all_tests + + max_exit_code = run_all_tests(stop_on_error=False) + + test_failed = max_exit_code != 0 + failure_message = f"Tests completed with exit code {max_exit_code}" if test_failed else None + + # Run post-processing once after all tests + try: + if test_failed and failure_message: + outcome = TestPhaseOutcome("FAILED", failure_message) + elif max_exit_code == 0: + outcome = TestPhaseOutcome("SUCCESS") + else: + outcome = TestPhaseOutcome("FAILED", f"exit_code={max_exit_code}") + + status = run_postprocess_after_test(env.BASE_ARTIFACT_DIR, test_outcome=outcome) + + if status is None or not status.get("success", False): + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + except Exception: + if max_exit_code == 0: + max_exit_code = 1 # Set exit code to 1 if post-processing failed but tests passed + + return max_exit_code @main.command() @@ -113,6 +187,7 @@ def post_cleanup(ctx) -> int: main.add_command( create_fournos_resolve_entrypoint( vault_list_funcs=[ + list_vaults, vault.phase_vault_list_all, caliper_export_list_vaults, caliper_export_list_optional_vaults, diff --git a/projects/llm_d/orchestration/cleanup_phase.py b/projects/llm_d/orchestration/cleanup_phase.py index 0bf1e0e86..1fac3327c 100644 --- a/projects/llm_d/orchestration/cleanup_phase.py +++ b/projects/llm_d/orchestration/cleanup_phase.py @@ -5,7 +5,7 @@ from projects.cluster.toolbox.cleanup_operators import ( main as cleanup_operators_command, ) -from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_resource_exists from projects.llm_d.toolbox.cleanup_test_resources import main as cleanup_test_resources_command logger = logging.getLogger(__name__) @@ -30,11 +30,7 @@ def cleanup_namespace(*, namespace: str | None = None) -> None: namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() inference_service_name = platform["inference_service"]["name"] - benchmark_job_names = runtime_config.get_benchmark_job_names() - # Multiple benchmarks may share a job_name (defaults to "guidellm-benchmark"); - # the broad labeled sweep below deletes every forge-labeled job/pod/pvc - # regardless, so a single representative name is enough for the named pass. - benchmark_name = benchmark_job_names[0] if benchmark_job_names else None + benchmark_name = runtime_config.get_benchmark_job_name() if not oc_resource_exists("namespace", namespace): return @@ -73,6 +69,8 @@ def cleanup_operators(*, dry_run: bool = False) -> None: dry_run=dry_run, ) + _cleanup_stale_crds(platform, dry_run=dry_run) + def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[dict]: """Identify which operators should be cleaned up based on configuration.""" @@ -112,3 +110,27 @@ def _identify_operators_to_delete(platform: dict, cleanup_config: dict) -> list[ logger.info(f"Identified {len(operators_to_delete)} operators for deletion") return operators_to_delete + + +def _cleanup_stale_crds(platform: dict, *, dry_run: bool = False) -> None: + """Delete CRDs listed under cleanup_crds in operator specs. + + Some operators (e.g. rhcl-operator) leave CRDs with stored API versions + that block reinstallation on the next run. + """ + operators = platform.get("operators", {}) + if not isinstance(operators, dict): + return + + for package_name, operator_spec in operators.items(): + if not isinstance(operator_spec, dict): + continue + for crd in operator_spec.get("cleanup_crds", []): + if not oc_resource_exists("crd", crd): + logger.info("CRD %s not present, skipping", crd) + continue + if dry_run: + logger.info("[DRY RUN] Would delete CRD %s (from %s)", crd, package_name) + continue + logger.info("Deleting stale CRD %s (from %s)", crd, package_name) + oc("delete", "crd", crd, "--ignore-not-found=true", "--timeout=60s", check=False) diff --git a/projects/llm_d/orchestration/cli.py b/projects/llm_d/orchestration/cli.py index 189b81ecc..362f73791 100755 --- a/projects/llm_d/orchestration/cli.py +++ b/projects/llm_d/orchestration/cli.py @@ -4,7 +4,6 @@ """ import logging -import sys import types from pathlib import Path @@ -16,13 +15,6 @@ logger = logging.getLogger(__name__) -def init(): - """Initialize LLM-D orchestration environment""" - env.init() - run.init() - config.init(Path(__file__).parent) - - @click.group() @click.option( "--preset", @@ -33,18 +25,63 @@ def init(): def main(ctx, preset): """CLI Operations.""" ctx.ensure_object(types.SimpleNamespace) - init() - if not preset: - return + env.init() + run.init() + + if preset: + config.write_variables_override(presets=preset) + + config.init(Path(__file__).parent) + +@main.command() +@click.option( + "--deployment-profile", + multiple=True, + help="The deployment profile(s) to deploy", +) +@click.option( + "--benchmark-key", + multiple=True, + help="The benchmark key(s) to launch.", +) +@click.option( + "--stop-on-error/--continue-on-error", + default=True, + help="Stop on the first test error (default: true)", +) +@click.pass_context +def deploy_and_test(ctx, deployment_profile, benchmark_key, stop_on_error) -> int: + """Test LLM-D with specified deployment profiles and benchmark keys.""" try: - for preset_name in preset: - logger.info(f"Applying preset: {preset_name}") - config.project.apply_preset(preset_name) - except ValueError as e: - logger.error(f"Failed to apply preset '{preset_name}': {e}") - sys.exit(1) + # Configure runtime settings based on provided options + if deployment_profile: + config.project.set_config("runtime.deployment_profile", list(deployment_profile)) + + if benchmark_key: + config.project.set_config("runtime.benchmark_key", list(benchmark_key)) + + # Initialize vaults for deployment + from projects.llm_d.orchestration.ci import init_vaults_for_phase + from projects.llm_d.orchestration.test_phase import run_all_tests + + # Test execution + logger.info("Starting test phase...") + init_vaults_for_phase("test") + + max_exit_code = run_all_tests(stop_on_error=stop_on_error) + + if max_exit_code != 0: + logger.error(f"Test phase completed with exit code {max_exit_code}") + return max_exit_code + + logger.info("All tests completed successfully!") + return 0 + + except Exception: + logger.exception("Deploy and test failed") + return 1 main.add_command(postprocess_command) diff --git a/projects/llm_d/orchestration/config.d/cpt.yaml b/projects/llm_d/orchestration/config.d/cpt.yaml new file mode 100644 index 000000000..bfb9b06c8 --- /dev/null +++ b/projects/llm_d/orchestration/config.d/cpt.yaml @@ -0,0 +1,5 @@ +kpi: + labels: + platform_name: null + gpu_type: null + test_harness: null diff --git a/projects/llm_d/orchestration/config.d/deployments.yaml b/projects/llm_d/orchestration/config.d/deployments.yaml index cc74273d8..a2b29c769 100644 --- a/projects/llm_d/orchestration/config.d/deployments.yaml +++ b/projects/llm_d/orchestration/config.d/deployments.yaml @@ -1,18 +1,60 @@ defaults: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d replicas: 1 tensor_parallelism: 1 - vllm_args: - - --max-model-len=4096 - - --gpu-memory-utilization=0.92 - - --trust-remote-code - - --no-enable-log-requests - - --enable-prefix-caching + vllm_extra: + args: + - --gpu-memory-utilization=0.92 + - --trust-remote-code + - --no-enable-log-requests + - --enable-prefix-caching -approximate-prefix-cache: - scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml +pd: + vllm_extra: + args: + - --block-size 128 + - --kv-transfer-config '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + env: + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + resources: + rdma/ib: '1' -precise-prefix-cache: - scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml +profiles: + approximate-prefix-cache: + scheduler_manifest: manifests/deployments/approximate-prefix-cache.yaml -distributed-default: - scheduler: {} + precise-prefix-cache: + scheduler_manifest: manifests/deployments/precise-prefix-cache.yaml + + distributed-default: + scheduler: {} + +# Template flavor configurations with name-based extraction +templates: + pd: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME # Extract main deployment tensor parallelism from profile name + + prefill: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + decode: + replicas: FROM_NAME + tensor_parallelism: FROM_NAME + + + intelligentrouting: + scheduler_manifest: {} # scheduler enabled with default config + tensor_parallelism: FROM_NAME + replicas: FROM_NAME + + simple: + scheduler_manifest: null # scheduler disabled + tensor_parallelism: FROM_NAME + replicas: FROM_NAME diff --git a/projects/llm_d/orchestration/config.d/platform.yaml b/projects/llm_d/orchestration/config.d/platform.yaml index 44a941275..3f911b06d 100644 --- a/projects/llm_d/orchestration/config.d/platform.yaml +++ b/projects/llm_d/orchestration/config.d/platform.yaml @@ -1,14 +1,14 @@ cluster: minimum_openshift_version: "4.19.9" - namespace: - name: forge-llm-d - prefix: llm-d - max_length: 63 gpu_node_label_selector: nvidia.com/gpu.present=true nfd_gpu_detection_labels: - feature.node.kubernetes.io/pci-10de.present - feature.node.kubernetes.io/pci-0302_10de.present - feature.node.kubernetes.io/pci-0300_10de.present + namespace: + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d operators: openshift-cert-manager-operator: @@ -41,6 +41,8 @@ operators: namespace: openshift-operators channel: stable source: redhat-operators + cleanup_crds: + - ztunnels.sailoperator.io rhods-operator: display_name: Red Hat OpenShift AI namespace: redhat-ods-operator @@ -50,10 +52,19 @@ operators: rhoai: custom_catalog: enabled: false - name: rhoai-custom + name: rhoai-catalog-dev namespace: openshift-marketplace image: null - display_name: Red Hat OpenShift AI Custom Catalog + display_name: Red Hat OpenShift AI + publisher: RHOAI Development Catalog + pull_secret: + vault: + name: psap-rhoai-rc + content: rhoai_rc.secret + staging_pull_secret: + vault: + name: psap-forge-staging-image-pull + content: .dockerconfigjson namespace: redhat-ods-applications datasciencecluster_name: default-dsc components: @@ -75,12 +86,18 @@ inference_service: name: llm-d template: manifests/llminferenceservice.yaml workload_deployment_name_suffix: -kserve + scheduler: + node_selector: + nvidia.com/gpu.present: "true" artifacts: capture_namespace_events: true smoke: pod_name: llm-d-smoke - client_image: curlimages/curl:8.11.1 + client_image: quay.io/curl/curl:8.21.0 endpoint_path: /v1/completions request_timeout_seconds: 60 + +preflight: + check_crds: true diff --git a/projects/llm_d/orchestration/config.d/project.yaml b/projects/llm_d/orchestration/config.d/project.yaml deleted file mode 100644 index 4d6b0b88f..000000000 --- a/projects/llm_d/orchestration/config.d/project.yaml +++ /dev/null @@ -1,3 +0,0 @@ -name: llm_d -args: - - smoke diff --git a/projects/llm_d/orchestration/config.d/runtime.yaml b/projects/llm_d/orchestration/config.d/runtime.yaml index 6dd9496ca..bea48e83a 100644 --- a/projects/llm_d/orchestration/config.d/runtime.yaml +++ b/projects/llm_d/orchestration/config.d/runtime.yaml @@ -4,4 +4,22 @@ deployment_profile: approximate-prefix-cache smoke_request_key: default benchmark_key: null job_name: null -namespace_override: null +namespace: forge-llm-d +kserve: + dry_run: false + wait_readiness: true + wait_long_scheduling: false + reuse_existing: false + delete_all_on_start: true +kueue: + enabled: false + queue_name: default-queue + local_queue_manifest: manifests/kueue-localqueue.yaml + disable_above_n_gpus: null +router: + template: + file: manifests/scheduler_config/__router_template__.yaml + placeholder: __ENDPOINT_PICKER_CONFIG__ + epp: manifests/scheduler_config + +run_test_finalizers: true diff --git a/projects/llm_d/orchestration/config.d/workloads.yaml b/projects/llm_d/orchestration/config.d/workloads.yaml index 5136666c9..0e29ea5eb 100644 --- a/projects/llm_d/orchestration/config.d/workloads.yaml +++ b/projects/llm_d/orchestration/config.d/workloads.yaml @@ -11,6 +11,10 @@ smoke_requests: temperature: 0.7 benchmarks: + default: + vllm_args: + max_model_len: 4096 + short: timeout_seconds: 900 outputs: json @@ -31,6 +35,8 @@ benchmarks: max_seconds: 600 heavy-heterogeneous: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent @@ -39,6 +45,8 @@ benchmarks: max_seconds: 600 multi-turn: + vllm_args: + max_model_len: 40960 args: backend_type: openai_http rate_type: concurrent diff --git a/projects/llm_d/orchestration/config.yaml b/projects/llm_d/orchestration/config.yaml index 56265eb55..3ced4607f 100644 --- a/projects/llm_d/orchestration/config.yaml +++ b/projects/llm_d/orchestration/config.yaml @@ -48,7 +48,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [] report_group: all visualize_config: projects/guidellm/postprocess/guidellm/visualize-groups.yaml @@ -66,11 +66,6 @@ caliper: enabled: true output_dir: ai_data - analyze: - enabled: true - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke @@ -78,7 +73,6 @@ caliper: vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials - prefix: llm_d/ import: enabled: true output_dir: historical_data @@ -94,9 +88,11 @@ caliper: include_kpis_json: true include_ai_data: true - analyse_kpis: - enabled: false - output: kpi_analysis.json + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json agentic: enabled: true diff --git a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml deleted file mode 100644 index 7e4e4ff1a..000000000 --- a/projects/llm_d/orchestration/manifests/deployments/precise-prefix-cache.yaml +++ /dev/null @@ -1,73 +0,0 @@ -scheduler: - template: - containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. - # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with - # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead - # of the older precise-prefix-cache-scorer-only configuration kept here. - - type: single-profile-handler - - type: precise-prefix-cache-scorer - parameters: - indexerConfig: - tokenProcessorConfig: - blockSize: 64 - hashSeed: "42" - tokenizersPoolConfig: - hf: - tokenizersCacheDir: /tmp/tokenizers - - type: kv-cache-utilization-scorer - - type: queue-scorer - - type: max-score-picker - schedulingProfiles: - - name: default - plugins: - - pluginRef: precise-prefix-cache-scorer - weight: 3.0 - - pluginRef: kv-cache-utilization-scorer - weight: 2.0 - - pluginRef: queue-scorer - weight: 2.0 - - pluginRef: max-score-picker - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} diff --git a/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml new file mode 100644 index 000000000..d2232815c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/kueue-localqueue.yaml @@ -0,0 +1,8 @@ +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + name: "QUEUE_NAME" + namespace: "NAMESPACE" +spec: + clusterQueue: "QUEUE_NAME" + stopPolicy: None diff --git a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml index 788170f0d..ba27d5d27 100644 --- a/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml +++ b/projects/llm_d/orchestration/manifests/leaderworkersetoperator.yaml @@ -5,4 +5,4 @@ metadata: spec: logLevel: Normal managementState: Managed - operatorLogLevel: Normal \ No newline at end of file + operatorLogLevel: Normal diff --git a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml index cff616f84..822fcdcbc 100644 --- a/projects/llm_d/orchestration/manifests/llminferenceservice.yaml +++ b/projects/llm_d/orchestration/manifests/llminferenceservice.yaml @@ -16,47 +16,7 @@ spec: scheduler: template: containers: - - name: main - env: - - name: TOKENIZER_CACHE_DIR - value: /tmp/tokenizer-cache - - name: HF_HOME - value: /tmp/tokenizer-cache - - name: TRANSFORMERS_CACHE - value: /tmp/tokenizer-cache - - name: XDG_CACHE_HOME - value: /tmp - args: - - --cert-path - - /var/run/kserve/tls - - --pool-group - - inference.networking.x-k8s.io - - --pool-name - - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" - - --pool-namespace - - "{{ .ObjectMeta.Namespace }}" - - --zap-encoder - - json - - --grpc-port - - "9002" - - --grpc-health-port - - "9003" - - --secure-serving - - --model-server-metrics-scheme - - https - - --config-text - volumeMounts: - - name: tokenizer-cache - mountPath: /tmp/tokenizer-cache - - name: cachi2-cache - mountPath: /cachi2 - volumes: - - name: tokenizer-cache - emptyDir: {} - - name: cachi2-cache - emptyDir: {} - nodeSelector: - nvidia.com/gpu.present: "true" + - name: main route: {} gateway: {} template: diff --git a/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml new file mode 100644 index 000000000..591e1e378 --- /dev/null +++ b/projects/llm_d/orchestration/manifests/rhoai-registry-icsp.yaml @@ -0,0 +1,15 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: ImageContentSourcePolicy +metadata: + name: rhoai-registry +spec: + repositoryDigestMirrors: + - source: registry.redhat.io/rhoai + mirrors: + - quay.io/rhoai + - source: registry.redhat.io/rhaii + mirrors: + - registry.stage.redhat.io/rhaii + - source: registry.redhat.io/rhaii-early-access + mirrors: + - registry.stage.redhat.io/rhaii-early-access diff --git a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml similarity index 66% rename from projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml rename to projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml index ae62e49d4..8c33cbd30 100644 --- a/projects/llm_d/orchestration/manifests/deployments/approximate-prefix-cache.yaml +++ b/projects/llm_d/orchestration/manifests/scheduler_config/__router_template__.yaml @@ -30,22 +30,9 @@ scheduler: - --model-server-metrics-scheme - https - --config-text - - | - apiVersion: inference.networking.x-k8s.io/v1alpha1 - kind: EndpointPickerConfig - plugins: - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: prefix-cache-scorer - schedulingProfiles: - - name: default - plugins: - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 + + - __ENDPOINT_PICKER_CONFIG__ + volumeMounts: - name: tokenizer-cache mountPath: /tmp/tokenizer-cache diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml new file mode 100644 index 000000000..ea0cf650c --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/approximate-prefix-cache.yaml @@ -0,0 +1,15 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + - type: queue-scorer + - type: kv-cache-utilization-scorer + - type: prefix-cache-scorer +schedulingProfiles: + - name: default + plugins: + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: prefix-cache-scorer + weight: 3.0 diff --git a/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml new file mode 100644 index 000000000..20e8895bf --- /dev/null +++ b/projects/llm_d/orchestration/manifests/scheduler_config/precise-prefix-cache.yaml @@ -0,0 +1,30 @@ +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: + # TODO: Align this profile with the upstream precise-prefix-cache-routing guide. + # Upstream now uses precise-prefix-cache-producer + prefix-cache-scorer, with + # vLLM KV events published over ZMQ (KV_EVENTS_ENDPOINT=tcp://*:5556), instead + # of the older precise-prefix-cache-scorer-only configuration kept here. + - type: single-profile-handler + - type: precise-prefix-cache-scorer + parameters: + indexerConfig: + tokenProcessorConfig: + blockSize: 64 + hashSeed: "42" + tokenizersPoolConfig: + hf: + tokenizersCacheDir: /tmp/tokenizers + - type: kv-cache-utilization-scorer + - type: queue-scorer + - type: max-score-picker +schedulingProfiles: + - name: default + plugins: + - pluginRef: precise-prefix-cache-scorer + weight: 3.0 + - pluginRef: kv-cache-utilization-scorer + weight: 2.0 + - pluginRef: queue-scorer + weight: 2.0 + - pluginRef: max-score-picker diff --git a/projects/llm_d/orchestration/phase_inputs.py b/projects/llm_d/orchestration/phase_inputs.py index 70f8b4387..2fbcc62ca 100644 --- a/projects/llm_d/orchestration/phase_inputs.py +++ b/projects/llm_d/orchestration/phase_inputs.py @@ -21,7 +21,6 @@ class PrepareModelCacheInputs: artifact_dir: Path preset_name: str namespace: str - namespace_is_managed: bool model_key: str model: dict[str, Any] model_cache: dict[str, Any] @@ -33,7 +32,6 @@ class PrepareInputs: config_dir: Path preset_name: str namespace: str - namespace_is_managed: bool platform: dict[str, Any] model_key: str model: dict[str, Any] @@ -83,7 +81,6 @@ def write_prepare_inputs_from_prepare(inputs: PrepareInputs) -> Path: "config_dir": str(inputs.config_dir), "preset_name": inputs.preset_name, "namespace": inputs.namespace, - "namespace_is_managed": inputs.namespace_is_managed, "platform": inputs.platform, "model_key": inputs.model_key, "model": inputs.model, @@ -125,7 +122,6 @@ def load_prepare_model_cache_inputs(path: str | Path) -> PrepareModelCacheInputs artifact_dir=Path(payload["artifact_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], model_key=payload["model_key"], model=payload["model"], model_cache=payload["model_cache"], @@ -137,7 +133,6 @@ def build_prepare_model_cache_inputs( artifact_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, model_key: str, model: dict[str, Any], model_cache: dict[str, Any], @@ -146,7 +141,6 @@ def build_prepare_model_cache_inputs( artifact_dir=Path(artifact_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, model_key=model_key, model=model, model_cache=model_cache, @@ -160,7 +154,6 @@ def load_prepare_inputs(path: str | Path) -> PrepareInputs: config_dir=Path(payload["config_dir"]), preset_name=payload["preset_name"], namespace=payload["namespace"], - namespace_is_managed=payload["namespace_is_managed"], platform=payload["platform"], model_key=payload["model_key"], model=payload["model"], @@ -175,7 +168,6 @@ def build_prepare_inputs( config_dir: str | Path, preset_name: str, namespace: str, - namespace_is_managed: bool, platform: dict[str, Any], model_key: str, model: dict[str, Any], @@ -187,7 +179,6 @@ def build_prepare_inputs( config_dir=Path(config_dir), preset_name=preset_name, namespace=namespace, - namespace_is_managed=namespace_is_managed, platform=platform, model_key=model_key, model=model, @@ -274,7 +265,6 @@ def prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> PrepareMod artifact_dir=inputs.artifact_dir, preset_name=inputs.preset_name, namespace=inputs.namespace, - namespace_is_managed=inputs.namespace_is_managed, model_key=inputs.model_key, model=inputs.model, model_cache=inputs.model_cache, @@ -290,7 +280,6 @@ def write_prepare_model_cache_inputs_from_prepare(inputs: PrepareInputs) -> Path "artifact_dir": str(cache_inputs.artifact_dir), "preset_name": cache_inputs.preset_name, "namespace": cache_inputs.namespace, - "namespace_is_managed": cache_inputs.namespace_is_managed, "model_key": cache_inputs.model_key, "model": cache_inputs.model, "model_cache": cache_inputs.model_cache, diff --git a/projects/llm_d/orchestration/pr_args.py b/projects/llm_d/orchestration/pr_args.py index 6811a4726..7c11fd048 100644 --- a/projects/llm_d/orchestration/pr_args.py +++ b/projects/llm_d/orchestration/pr_args.py @@ -2,14 +2,22 @@ """ llm_d Project PR Arguments Parser -llm_d relies on the framework preset mechanism from `/test fournos llm_d PRESET`. -No llm_d-specific PR directives are currently required. +Parses llm_d-specific directives from PR trigger comments. + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 """ from __future__ import annotations +import logging from typing import Any +from projects.rhoai.library.pr_args import get_rhoai_directive_handlers + +logger = logging.getLogger(__name__) + def get_supported_llm_d_directives() -> dict[str, str]: """ @@ -31,5 +39,21 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st Returns: Tuple of (configuration overrides dict, list of parsed directive lines) """ - _ = comment_text - return {}, [] + directive_handlers = get_rhoai_directive_handlers() + config_overrides: dict[str, Any] = {} + parsed_directives: list[str] = [] + + for raw_line in comment_text.splitlines(): + line = raw_line.strip() + if not line: + continue + + for prefix, handler in directive_handlers.items(): + if line.startswith(prefix): + result = handler(line) + config_overrides.update(result) + parsed_directives.append(line) + logger.info("Parsed llm_d directive: %s", line) + break + + return config_overrides, parsed_directives diff --git a/projects/llm_d/orchestration/preflight_phase.py b/projects/llm_d/orchestration/preflight_phase.py index e6c0b17c0..86ab3eddd 100644 --- a/projects/llm_d/orchestration/preflight_phase.py +++ b/projects/llm_d/orchestration/preflight_phase.py @@ -3,22 +3,19 @@ import logging from projects.core.dsl.utils.k8s import oc_resource_exists +from projects.core.library import ci as ci_lib from projects.llm_d.orchestration import runtime_config logger = logging.getLogger(__name__) -def run() -> int: - """Run preflight checks before testing phase. - - Validates that required CRDs exist in the cluster: - - leaderworkersets.leaderworkerset.x-k8s.io - - llminferenceservices.serving.kserve.io +def check_crds() -> bool: + """Check that required CRDs exist in the cluster. Returns: - 0 on success, non-zero on failure + True if all CRDs are present, False otherwise """ - logger.info("Starting preflight checks") + logger.info("Checking required CRDs") # Get required CRDs from platform configuration platform = runtime_config.get_platform_config() @@ -36,10 +33,49 @@ def run() -> int: logger.info(f"CRD found: {crd_name}") if missing_crds: - logger.error( - f"Preflight check failed - missing {len(missing_crds)} required CRDs: {', '.join(missing_crds)}" + error_message = f"Missing {len(missing_crds)} required CRDs:\n" + "\n".join( + f"- {crd}" for crd in missing_crds ) - return 1 + logger.error(f"CRD check failed - {error_message}") + ci_lib.add_notification_file("MISSING_CRDS", error_message) + return False + + logger.info("CRD check passed - all required CRDs are available") + return True + + +def run() -> int: + """Run preflight checks before testing phase. + + Validates that required components exist in the cluster based on + platform configuration settings. - logger.info("Preflight checks completed successfully - all required CRDs are available") - return 0 + Returns: + 0 on success, non-zero on failure + """ + logger.info("Starting preflight checks") + + platform = runtime_config.get_platform_config() + preflight_config = platform.get("preflight", {}) + + # Track overall success across all checks + all_checks_passed = True + + # Run CRD checks if enabled + if preflight_config.get("check_crds", True): + if not check_crds(): + all_checks_passed = False + else: + logger.info("CRD checks disabled by platform configuration") + + # Future checks can be added here: + # if preflight_config.get("check_operators", True): + # if not check_operators(): + # all_checks_passed = False + + if all_checks_passed: + logger.info("Preflight checks completed successfully") + return 0 + else: + logger.error("One or more preflight checks failed") + return 1 diff --git a/projects/llm_d/orchestration/prepare_phase.py b/projects/llm_d/orchestration/prepare_phase.py index 138c24509..309b2a714 100644 --- a/projects/llm_d/orchestration/prepare_phase.py +++ b/projects/llm_d/orchestration/prepare_phase.py @@ -3,11 +3,9 @@ import hashlib import json import logging -from typing import Any +from pathlib import Path from projects.cluster.toolbox.bootstrap_lws_operator import main as bootstrap_lws_operator -from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator -from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name from projects.core.dsl.utils.k8s import ( @@ -29,25 +27,20 @@ run as capture_prepare_state_toolbox_run, ) from projects.llm_d.toolbox.ensure_gateway import main as ensure_gateway_command +from projects.rhoai.library.deploy import ( + ensure_operator_subscription, + operator_spec_by_package, +) +from projects.rhoai.library.deploy import ( + prepare_rhoai_operator as rhoai_prepare_rhoai_operator, +) from projects.rhoai.toolbox.apply_datasciencecluster import main as apply_datasciencecluster_command from projects.rhoai.toolbox.wait_datasciencecluster_ready import ( main as wait_datasciencecluster_ready_command, ) logger = logging.getLogger(__name__) - - -def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: - operators = platform["operators"] - if isinstance(operators, dict): - if package in operators: - return {"package": package, **operators[package]} - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") - - for operator_spec in operators: - if operator_spec["package"] == package: - return operator_spec - raise KeyError(f"Unknown operator package in llm_d platform config: {package}") +RHOAI_ICSP_MANIFEST = Path(__file__).resolve().parent / "manifests" / "rhoai-registry-icsp.yaml" def verify_oc_access() -> None: @@ -76,54 +69,9 @@ def verify_cluster_version() -> None: ) -def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: - return cluster_deploy_operator.run( - package_name=operator_spec["package"], - target_namespace=operator_spec["namespace"], - source_name=operator_spec["source"], - channel=operator_spec["channel"], - source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), - display_name=operator_spec.get("display_name", operator_spec["package"]), - artifact_dirname_suffix=f"_{operator_spec['package']}", - ) - - -def deploy_rhoai_custom_catalog(*, rhoai: dict) -> int: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - logger.info("RHOAI custom catalog disabled; using default catalog source") - return 0 - - if not custom_catalog.get("image"): - raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") - - return deploy_custom_catalog.run( - catalog_source_name=custom_catalog["name"], - catalog_namespace=custom_catalog["namespace"], - catalog_image=custom_catalog["image"], - display_name=custom_catalog.get("display_name", custom_catalog["name"]), - ) - - -def rhoai_operator_spec( - *, - rhoai: dict, - operator_spec: dict[str, str], -) -> dict[str, str]: - custom_catalog = rhoai["custom_catalog"] - if not custom_catalog["enabled"]: - return operator_spec - - updated_spec = dict(operator_spec) - updated_spec["source"] = custom_catalog["name"] - updated_spec["source_namespace"] = custom_catalog["namespace"] - return updated_spec - - -def prepare_rhcl_operator() -> None: - platform = runtime_config.get_platform_config() - operator_spec = operator_spec_by_package(platform, "rhcl-operator") - ensure_operator_subscription(operator_spec) +def apply_rhoai_icsp() -> None: + logger.info("Applying RHOAI ICSP manifest: %s", RHOAI_ICSP_MANIFEST) + oc("apply", "-f", str(RHOAI_ICSP_MANIFEST)) def prepare_cert_manager() -> None: @@ -179,20 +127,10 @@ def prepare_gpu_operator() -> None: def prepare_rhoai_operator() -> None: platform = runtime_config.get_platform_config() - prepare_rhcl_operator() - deploy_rhoai_custom_catalog(rhoai=platform["rhoai"]) - operator_spec = operator_spec_by_package(platform, "rhods-operator") - operator_spec = rhoai_operator_spec(rhoai=platform["rhoai"], operator_spec=operator_spec) - ensure_operator_subscription(operator_spec) - ensure_required_crds_before_dsc() - - -def ensure_required_crds_before_dsc() -> None: - platform = runtime_config.get_platform_config() - rhoai = platform["rhoai"] - wait_for_crds_command.run( - crd_names=rhoai["required_crds_before_dsc"], - display_name="RHOAI pre-DSC CRDs", + rhoai_prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=apply_rhoai_icsp, ) @@ -345,12 +283,11 @@ def prepare_model_cache() -> None: common_args = { "namespace": namespace, - "namespace_is_managed": runtime_config.get_namespace_is_managed(), "model_key": model_slug, "model_uri": model_uri, "pvc_size": model_cache["pvc"]["size"], "access_mode": model_cache["pvc"]["access_mode"], - "storage_class_name": model_cache["pvc"].get("storage_class_name"), + "storage_class_name": model_cache["pvc"]["storage_class_name"], "pvc_name_prefix": pvc_name_prefix, "model_directory_name": model_cache["pvc"]["model_directory_name"], } diff --git a/projects/llm_d/orchestration/presets.d/benchmarks.yaml b/projects/llm_d/orchestration/presets.d/benchmarks.yaml new file mode 100644 index 000000000..e9dc9e475 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/benchmarks.yaml @@ -0,0 +1,10 @@ +__multiple: true + +concurrent-1k-1k: + runtime.benchmark_key: concurrent-1k-1k + +heavy-heterogeneous: + runtime.benchmark_key: heavy-heterogeneous + +multi-turn: + runtime.benchmark_key: multi-turn diff --git a/projects/llm_d/orchestration/presets.d/cks.yaml b/projects/llm_d/orchestration/presets.d/cks.yaml new file mode 100644 index 000000000..b03451fce --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cks.yaml @@ -0,0 +1,65 @@ +__multiple: true + +cks: + runtime.namespace: kpouget-dev + platform.cluster.namespace.labels: {} + + runtime.kserve.wait_readiness: false + runtime.kueue.enabled: true + runtime.kueue.queue_name: llm-d-cluster-queue + runtime.kueue.disable_above_n_gpus: 8 + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + platform.gateway.status_address_name: gateway-internal + platform.preflight.check_crds: false + runtime.kserve.wait_long_scheduling: true + platform.inference_service.scheduler.node_selector: + gpu.nvidia.com/class: H200 + + cpt.kpi.labels.platform_name: CKS + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + +cks-intelligent: + extends: + - cks + - cpt + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-pd: + extends: + - cks + - cpt + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +cks-smoke: + extends: + - cks + #- llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - intelligentrouting-tp1-x2 + - simple-tp1-x1 + - pd-d.x1-p.tp1-d.tp1-p.x1 + + runtime.benchmark_key: + - short diff --git a/projects/llm_d/orchestration/presets.d/cpt.yaml b/projects/llm_d/orchestration/presets.d/cpt.yaml new file mode 100644 index 000000000..ce7ce6859 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/cpt.yaml @@ -0,0 +1,13 @@ +__multiple: true + +# CPT reference profiles +cpt-reference-flavors: + extends: + - gpt-oss + runtime.deployment_profile: + - pd-d.x2-p.tp1-d.tp4-p.x8 + - intelligentrouting-tp4-x4 + - simple-tp4-x4 + +cpt: + caliper.export.backend.mlflow.config.experiment: cpt-llm-d diff --git a/projects/llm_d/orchestration/presets.d/janus.yaml b/projects/llm_d/orchestration/presets.d/janus.yaml new file mode 100644 index 000000000..ed629be57 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/janus.yaml @@ -0,0 +1,56 @@ +__multiple: true + +janus: + runtime.namespace: kpouget-dev + + # Use ReadWriteMany for model cache PVC in CKS environment + model_cache.pvc.access_mode: ReadWriteMany + model_cache.pvc.storage_class_name: nfs + deployments.pd.resources: composite.dra.io/gpu-nic-pair + platform.gateway.status_address_name: null + platform.gateway.status_address_name: gateway-internal + + cpt.kpi.labels.platform_name: OCP + cpt.kpi.labels.gpu_type: H200 + cpt.kpi.labels.test_harness: xks + +janus-intelligent: + extends: + - janus + - cpt + - llama3-70b + + runtime.deployment_profile: + - intelligentrouting-tp2-x4 + - simple-tp2-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-pd: + extends: + - janus + - cpt + - gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + - simple-tp4-x4 + - intelligent-tp4-x4 + + runtime.benchmark_key: + - heavy-heterogeneous + - multi-turn + +janus-smoke: + extends: + - janus + - llama3-70b + #- gpt-oss + + runtime.deployment_profile: + - pd-p.tp1-p.x8-d.tp4-d.x2 + + runtime.benchmark_key: + - short diff --git a/projects/llm_d/orchestration/presets.d/models.yaml b/projects/llm_d/orchestration/presets.d/models.yaml new file mode 100644 index 000000000..995860310 --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/models.yaml @@ -0,0 +1,14 @@ +__multiple: true + +gpt-oss: + runtime.model_name: openai/gpt-oss-120b + runtime.deployment_profile: distributed-default + runtime.smoke_request_key: default + model_cache.pvc.size: 300Gi + workloads.args: + request_type: text_completions + +llama3-70b: + runtime.model_name: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic + model_cache.pvc.size: 100Gi + runtime.smoke_request_key: default diff --git a/projects/llm_d/orchestration/presets.d/presets.yaml b/projects/llm_d/orchestration/presets.d/presets.yaml index 34d0609e2..6b42bb701 100644 --- a/projects/llm_d/orchestration/presets.d/presets.yaml +++ b/projects/llm_d/orchestration/presets.d/presets.yaml @@ -1,15 +1,12 @@ __multiple: true deployment-approximate-prefix-cache: - runtime.selected_preset: deployment-approximate-prefix-cache runtime.deployment_profile: approximate-prefix-cache deployment-precise-prefix-cache: - runtime.selected_preset: deployment-precise-prefix-cache runtime.deployment_profile: precise-prefix-cache deployment-distributed-default: - runtime.selected_preset: deployment-distributed-default runtime.deployment_profile: distributed-default smoke-base: @@ -21,7 +18,6 @@ smoke: extends: - smoke-base - deployment-approximate-prefix-cache - runtime.selected_preset: smoke runtime.benchmark_key: short platform.gateway.status_address_name: gateway-internal @@ -29,35 +25,23 @@ smoke-precise: extends: - smoke-base - deployment-precise-prefix-cache - runtime.selected_preset: smoke-precise smoke-default-scheduler: extends: - smoke-base - deployment-distributed-default - runtime.selected_preset: smoke-default-scheduler benchmark-short: extends: - deployment-approximate-prefix-cache - runtime.selected_preset: benchmark-short runtime.model_name: meta-llama/Llama-3.1-8B-Instruct runtime.smoke_request_key: default runtime.benchmark_key: short gpt-oss-120b-inference-scheduling-release: - runtime.selected_preset: gpt-oss-120b-inference-scheduling-release - runtime.model_name: openai/gpt-oss-120b - runtime.deployment_profile: distributed-default - runtime.smoke_request_key: default + extends: + - gpt-oss runtime.benchmark_key: - concurrent-1k-1k - heavy-heterogeneous - multi-turn - model_cache.pvc.size: 300Gi - workloads.args: - request_type: text_completions - -cks: - extends: - - smoke diff --git a/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml new file mode 100644 index 000000000..d1943a32c --- /dev/null +++ b/projects/llm_d/orchestration/presets.d/rhoai-rc.yaml @@ -0,0 +1,7 @@ +rhoai-rc: + platform.rhoai.custom_catalog.enabled: true + platform.operators.rhods-operator.channel: beta + +rhoai-3.5-ea.2: + extends: [rhoai-rc] + platform.rhoai.custom_catalog.image: quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310 diff --git a/projects/llm_d/orchestration/render_inference_service.py b/projects/llm_d/orchestration/render_inference_service.py index e823a867d..b27456e9c 100644 --- a/projects/llm_d/orchestration/render_inference_service.py +++ b/projects/llm_d/orchestration/render_inference_service.py @@ -2,12 +2,16 @@ import copy import hashlib +import logging from pathlib import Path from typing import Any import yaml from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.library import config + +logger = logging.getLogger(__name__) def _load_yaml(path: Path) -> Any: @@ -15,6 +19,44 @@ def _load_yaml(path: Path) -> Any: return yaml.safe_load(handle) +def _create_model_cache_spec( + model_cache: dict[str, Any], + source_uri: str, + source_scheme: str, + model_slug: str, + namespace: str, +) -> dict[str, Any] | None: + """Create model cache specification if caching is enabled and applicable.""" + if not model_cache.get("enabled", False) or source_uri.startswith(("pvc://", "pvc+hf://")): + return None + + pvc_defaults = model_cache["pvc"] + pvc_prefix = pvc_defaults["name_prefix"] + cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] + pvc_name = truncate_k8s_name( + f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" + ) + model_path = pvc_defaults["model_directory_name"] + + return { + "source_uri": source_uri, + "source_scheme": source_scheme, + "cache_key": cache_key, + "namespace": namespace, + "pvc_name": pvc_name, + "pvc_size": pvc_defaults["size"], + "access_mode": pvc_defaults["access_mode"], + "storage_class_name": pvc_defaults.get("storage_class_name"), + "model_path": model_path, + "model_uri": f"pvc://{pvc_name}/{model_path}", + "marker_filename": model_cache["marker_filename"], + "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", + "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), + "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), + "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), + } + + def render_inference_service_from_parts( *, config_dir: str | Path, @@ -24,22 +66,36 @@ def render_inference_service_from_parts( model_slug: str, deployment_profile: dict[str, Any], model_cache: dict[str, Any], + deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render an llm_d-owned LLMInferenceService manifest from concrete runtime inputs.""" template_path = Path(config_dir) / inference_service["template"] manifest = _load_yaml(template_path) + # Check if this is a P/D deployment + is_pd_deployment = "prefill" in deployment_profile + name = inference_service["name"] + if deployment_profile_name: + name = f"{name}-{deployment_profile_name}" + # Normalize name to be Kubernetes compliant and limit to 25 characters + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) manifest["metadata"]["name"] = name manifest["metadata"]["namespace"] = namespace manifest["metadata"].setdefault("labels", {}) manifest["metadata"]["labels"].update( - { - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - } + config.project.get_config("deployments.defaults.labels") or {} ) + # Add deployment profile name as annotation for testing + manifest["metadata"].setdefault("annotations", {}) + if deployment_profile_name: + manifest["metadata"]["annotations"]["forge.openshift.io/deployment-profile"] = ( + deployment_profile_name + ) + if model_name.startswith("oci://"): source_uri = model_name source_scheme = "oci" @@ -50,70 +106,84 @@ def render_inference_service_from_parts( source_uri = f"hf://{model_name}" source_scheme = "hf" - cache_spec = None - if model_cache.get("enabled", False) and not source_uri.startswith(("pvc://", "pvc+hf://")): - pvc_defaults = model_cache["pvc"] - pvc_prefix = pvc_defaults["name_prefix"] - cache_key = hashlib.sha256(source_uri.encode("utf-8")).hexdigest()[:10] - pvc_name = truncate_k8s_name( - f"{pvc_prefix}-{slugify_identifier(model_slug, max_length=32)}-{cache_key}" - ) - model_path = pvc_defaults["model_directory_name"] - - cache_spec = { - "source_uri": source_uri, - "source_scheme": source_scheme, - "cache_key": cache_key, - "namespace": namespace, - "pvc_name": pvc_name, - "pvc_size": pvc_defaults["size"], - "access_mode": pvc_defaults["access_mode"], - "storage_class_name": pvc_defaults.get("storage_class_name"), - "model_path": model_path, - "model_uri": f"pvc://{pvc_name}/{model_path}", - "marker_filename": model_cache["marker_filename"], - "marker_path": f"/cache/{model_path}/{model_cache['marker_filename']}", - "download_job_name": truncate_k8s_name(f"{pvc_name}-download"), - "hf_token_secret_name": model_cache["hf"].get("token_secret_name"), - "hf_token_secret_key": model_cache["hf"].get("token_secret_key"), - } + cache_spec = _create_model_cache_spec( + model_cache=model_cache, + source_uri=source_uri, + source_scheme=source_scheme, + model_slug=model_slug, + namespace=namespace, + ) - manifest["spec"]["replicas"] = deployment_profile["replicas"] manifest["spec"]["model"]["uri"] = cache_spec["model_uri"] if cache_spec else source_uri manifest["spec"]["model"]["name"] = model_slug - serving_container = manifest["spec"]["template"]["containers"][0] - serving_container["resources"] = _build_serving_resources(deployment_profile) - if deployment_profile.get("serving_image"): - serving_container["image"] = deployment_profile["serving_image"] - vllm_args = _build_vllm_args(deployment_profile.get("vllm_args", {})) - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) - if not _has_cli_arg(vllm_args, "tensor-parallel-size"): - vllm_args.append(f"--tensor-parallel-size={tensor_parallelism}") - if vllm_args: - serving_container["args"] = vllm_args - - scheduler = deployment_profile.get("scheduler", {}) - if scheduler is None: - manifest["spec"]["router"].pop("scheduler", None) + if is_pd_deployment: + rendered_manifest = _render_pd_deployment( + manifest, deployment_profile, deployment_profile_name, workload + ) else: - manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) - if deployment_profile.get("router_image"): - manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( - deployment_profile["router_image"] - ) + rendered_manifest = _render_standard_deployment( + manifest, deployment_profile, deployment_profile_name, workload + ) - return manifest + # Apply Kueue configuration if enabled + _apply_kueue_configuration(rendered_manifest, deployment_profile) + + return rendered_manifest + + +def handle_pd_resources( + base_resources: dict[str, Any], + deployment_profile: dict[str, Any], + is_prefill: bool = False, +) -> None: + """Handle P/D resource configuration, including DRA support. + + Args: + base_resources: Base resources dict to modify in-place + deployment_profile: Deployment profile configuration + is_prefill: Whether this is for a prefill container + """ + pd_resources = config.project.get_config("deployments.pd.resources") + + if pd_resources == "composite.dra.io/gpu-nic-pair": + # Handle DRA (Dynamic Resource Allocation) case + if is_prefill: + # For prefill pods, use prefill tensor parallelism + tensor_parallelism = deployment_profile["prefill"]["tensor_parallelism"] + else: + # For decode pods, use decode tensor parallelism + tensor_parallelism = deployment_profile["decode"]["tensor_parallelism"] + + # Override GPU resources for DRA + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + # Set nvidia.com/gpu to 0 and add composite DRA resource + base_resources[bound]["nvidia.com/gpu"] = "0" + base_resources[bound][pd_resources] = str(tensor_parallelism) + elif isinstance(pd_resources, dict): + # Normal case: pd_resources is a dictionary to merge + for bound in ("requests", "limits"): + if bound not in base_resources: + base_resources[bound] = {} + base_resources[bound].update(pd_resources) + elif pd_resources is not None: + # Handle unexpected pd_resources type + raise ValueError( + f"Unexpected type for deployments.pd.resources: {type(pd_resources)}, value: {pd_resources}" + ) + # If pd_resources is None, do nothing def _build_serving_resources(deployment_profile: dict[str, Any]) -> dict[str, Any]: - tensor_parallelism = str(deployment_profile["tensor_parallelism"]) + tensor_parallelism = deployment_profile["tensor_parallelism"] profile_resources = deployment_profile.get("resources", {}) rendered_resources: dict[str, Any] = {} for bound in ("requests", "limits"): source = profile_resources.get(bound, {}) - rendered_bound = {"nvidia.com/gpu": tensor_parallelism} + rendered_bound = {"nvidia.com/gpu": str(tensor_parallelism)} for resource_name in ("cpu", "memory"): value = source.get(resource_name) if value not in (None, ""): @@ -142,3 +212,385 @@ def _has_cli_arg(args: list[str], option_name: str) -> bool: prefix = f"--{option_name}=" bare = f"--{option_name}" return any(arg == bare or arg.startswith(prefix) for arg in args) + + +def _build_vllm_additional_args( + deployment_profile: dict[str, Any], + workload: dict[str, Any] | None = None, +) -> str: + """Build VLLM_ADDITIONAL_ARGS string based on deployment profile configuration. + + Args: + deployment_profile: The deployment profile configuration + workload: The workload configuration (merged benchmark config) + + Returns: + String suitable for VLLM_ADDITIONAL_ARGS environment variable + """ + + vllm_extra = deployment_profile.get("vllm_extra", {}) + vllm_deploy_args = _build_vllm_args(vllm_extra.get("args", {})) + + # Add workload vllm_args if available + if workload and "vllm_args" in workload: + workload_vllm_args = _build_vllm_args(workload["vllm_args"]) + vllm_deploy_args.extend(workload_vllm_args) + + return " ".join(vllm_deploy_args) + + +def _render_standard_deployment( + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Render standard (non-P/D) deployment configuration.""" + # Check if this is intelligent routing (scheduler_manifest exists) + scheduler = deployment_profile.get("scheduler") + has_scheduler_manifest = "scheduler_manifest" in deployment_profile + + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name + manifest["spec"]["replicas"] = deployment_profile["replicas"] + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["tensor_parallelism"]} + + serving_container = manifest["spec"]["template"]["containers"][0] + serving_container["resources"] = _build_serving_resources(deployment_profile) + if deployment_profile.get("serving_image"): + serving_container["image"] = deployment_profile["serving_image"] + + vllm_additional_args = _build_vllm_additional_args(deployment_profile, workload) + + # Add environment variable (don't set generic env vars or args) + if "env" not in serving_container: + serving_container["env"] = [] + + serving_container["env"].append({"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}) + + # Configure router/scheduler + has_scheduler_key = "scheduler" in deployment_profile + is_simple_deployment = not has_scheduler_key and not has_scheduler_manifest + + if is_simple_deployment: + # Simple deployments (no scheduler key, no scheduler_manifest) have no router section at all + manifest["spec"].pop("router", None) + elif scheduler is None: + # Some deployments might have router but no scheduler + manifest["spec"]["router"].pop("scheduler", None) + else: + # Configure scheduler for intelligent routing + manifest["spec"]["router"]["scheduler"] = copy.deepcopy(scheduler) + if deployment_profile.get("router_image"): + manifest["spec"]["router"]["scheduler"]["template"]["containers"][0]["image"] = ( + deployment_profile["router_image"] + ) + + return manifest + + +def _render_pd_deployment( + manifest: dict[str, Any], + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, + workload: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Render P/D (Prefill/Decode) deployment configuration.""" + from .runtime_config import get_decode_pod_count, get_prefill_pod_count + + # Set manifest name with deployment profile + name = f"llm-d-{deployment_profile_name}" + name = slugify_identifier(name) + name = truncate_k8s_name(name, max_length=25) + manifest["metadata"]["name"] = name + + # Configure prefill section + manifest["spec"]["prefill"] = { + "replicas": get_prefill_pod_count(), + "parallelism": {"tensor": deployment_profile["prefill"]["tensor_parallelism"]}, + "template": _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=True, workload=workload + ), + } + + # Configure main template (decode) + manifest["spec"]["replicas"] = get_decode_pod_count() + manifest["spec"]["parallelism"] = {"tensor": deployment_profile["decode"]["tensor_parallelism"]} + manifest["spec"]["template"] = _build_pd_pod_template( + deployment_profile, deployment_profile_name, is_prefill=False, workload=workload + ) + + return manifest + + +def _build_pd_pod_template( + deployment_profile: dict[str, Any], + deployment_profile_name: str | None = None, + is_prefill: bool = False, + workload: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build pod template for P/D deployment.""" + + # Get P/D extra configuration from deployments config + + pd_vllm_extra = config.project.get_config("deployments.pd.vllm_extra") + + # Build VLLM args with correct tensor parallelism + + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_vllm_args = _build_vllm_additional_args(prefill_profile, workload) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) + else: + # For decode pods, use main tensor parallelism + base_vllm_args = _build_vllm_additional_args(deployment_profile, workload) + + # Add P/D extra args + + pd_extra_args = pd_vllm_extra.get("args", []) + all_vllm_args = base_vllm_args.split() + pd_extra_args + + vllm_additional_args = " ".join(all_vllm_args) + + # Build base environment variables + base_env = [ + {"name": "VLLM_ADDITIONAL_ARGS", "value": vllm_additional_args}, + ] + + # Add P/D extra environment variables + pd_extra_env = pd_vllm_extra.get("env", []) + all_env = base_env + copy.deepcopy(pd_extra_env) + + # Build base resources with correct tensor parallelism + if is_prefill and deployment_profile_name: + # For prefill pods, use prefill tensor parallelism + from .runtime_config import _extract_value_from_profile_name + + try: + prefill_tp = _extract_value_from_profile_name( + deployment_profile_name, "prefill_tensor_parallelism" + ) + # Create modified profile with prefill tensor parallelism + prefill_profile = copy.deepcopy(deployment_profile) + prefill_profile["tensor_parallelism"] = prefill_tp + base_resources = _build_serving_resources(prefill_profile) + except ValueError: + # Fallback to main tensor parallelism if extraction fails + base_resources = _build_serving_resources(deployment_profile) + else: + # For decode pods, use main tensor parallelism + base_resources = _build_serving_resources(deployment_profile) + + # Handle P/D extra resources + handle_pd_resources(base_resources, deployment_profile, is_prefill) + + # Build container configuration + container = { + "name": "main", + "resources": base_resources, + "env": all_env, + } + + # Add serving image if specified + if deployment_profile.get("serving_image"): + container["image"] = deployment_profile["serving_image"] + + # Build pod template with anti-affinity for P/D deployments + component_type = "prefill" if is_prefill else "decode" + opposite_component = "decode" if is_prefill else "prefill" + + pod_template = { + "containers": [container], + "metadata": {"labels": {"app.kubernetes.io/component": component_type}}, + } + + # Add anti-affinity to prevent prefill and decode pods from landing on the same node + affinity = { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 100, + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + # Anti-affinity between prefill and decode pods + "app.kubernetes.io/component": opposite_component + } + }, + "topologyKey": "kubernetes.io/hostname", + }, + } + ] + } + } + + pod_template["affinity"] = affinity + + return pod_template + + +def _calculate_total_gpu_usage(deployment_profile: dict[str, Any]) -> int: + """Calculate the total GPU usage for a deployment profile. + + Args: + deployment_profile: The deployment profile configuration + + Returns: + Total number of GPUs required for this deployment + """ + # Check if this is a P/D deployment (has prefill/decode sections) + if "prefill" in deployment_profile and "decode" in deployment_profile: + # P/D deployment: sum up prefill and decode GPU usage + prefill_gpus = ( + deployment_profile["prefill"]["tensor_parallelism"] + * deployment_profile["prefill"]["replicas"] + ) + decode_gpus = ( + deployment_profile["decode"]["tensor_parallelism"] + * deployment_profile["decode"]["replicas"] + ) + return prefill_gpus + decode_gpus + else: + # Standard deployment: tensor_parallelism * replicas + tensor_parallelism = deployment_profile.get("tensor_parallelism", 1) + replicas = deployment_profile.get("replicas", 1) + return tensor_parallelism * replicas + + +def _apply_kueue_configuration( + manifest: dict[str, Any], deployment_profile: dict[str, Any] +) -> None: + """Apply Kueue annotations and labels to the ISVC manifest. + + Based on the implementation from topsail's test_llmd.py. + Can be enabled by setting runtime.kserve_use_kueue config. + + Args: + manifest: The Kubernetes manifest to modify + deployment_profile: The deployment profile configuration used to calculate GPU usage + """ + # Check if kueue annotations should be enabled + enable_kueue = config.project.get_config("runtime.kueue.enabled") + + if not enable_kueue: + return + + # Calculate total GPU usage from deployment profile + total_gpus = _calculate_total_gpu_usage(deployment_profile) + + # Check if we should skip Kueue due to high GPU usage + disable_above_n_gpus = config.project.get_config("runtime.kueue.disable_above_n_gpus") + if disable_above_n_gpus is not None and total_gpus > disable_above_n_gpus: + logger.info(f"Skipping Kueue labels: {total_gpus} GPUs > {disable_above_n_gpus} threshold") + return + + # Configure kueue settings + queue_name = config.project.get_config("runtime.kueue.queue_name") + kueue_config = { + "enabled": True, + "prefix": "kueue.x-k8s.io/", + "labels": {"queue-name": queue_name}, + "annotations": {"queue-name": queue_name}, + } + + # Get prefix for kueue labels/annotations + kueue_prefix = kueue_config.get("prefix", "kueue.x-k8s.io/") + + # Ensure metadata sections exist + if "metadata" not in manifest: + manifest["metadata"] = {} + if "labels" not in manifest["metadata"]: + manifest["metadata"]["labels"] = {} + if "annotations" not in manifest["metadata"]: + manifest["metadata"]["annotations"] = {} + + # Apply Kueue labels + kueue_labels = kueue_config.get("labels", {}) + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + manifest["metadata"]["labels"][full_label_key] = label_value + + # Apply Kueue annotations + kueue_annotations = kueue_config.get("annotations", {}) + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + manifest["metadata"]["annotations"][full_annotation_key] = annotation_value + + # Apply Kueue annotations to router scheduler if it exists + if ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ): + scheduler = manifest["spec"]["router"]["scheduler"] + + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} + + # Apply the same Kueue annotations to the scheduler + for annotation_key, annotation_value in kueue_annotations.items(): + full_annotation_key = f"{kueue_prefix}{annotation_key}" + scheduler["annotations"][full_annotation_key] = annotation_value + + # Apply the same Kueue labels to the scheduler + for label_key, label_value in kueue_labels.items(): + full_label_key = f"{kueue_prefix}{label_key}" + scheduler["labels"][full_label_key] = label_value + + # Calculate pod group total count: 1 scheduler + number of replicas + replicas = manifest.get("spec", {}).get("replicas", 1) + + # For P/D deployments, we need to account for prefill replicas too + prefill_replicas = 0 + if "spec" in manifest and "prefill" in manifest["spec"]: + prefill_replicas = manifest["spec"]["prefill"].get("replicas", 0) + + # Total: main replicas + prefill replicas + (1 scheduler if router exists) + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) + + scheduler_count = 1 if has_scheduler else 0 + pod_group_total_count = replicas + prefill_replicas + scheduler_count + + manifest["metadata"]["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) + + # Add required pod-group-name label using the ISVC name + pod_group_name = manifest["metadata"]["name"] + manifest["metadata"]["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name + + # Also add required Kueue annotations/labels to scheduler if it exists + if has_scheduler: + scheduler = manifest["spec"]["router"]["scheduler"] + + # Ensure annotations and labels exist on scheduler + if "annotations" not in scheduler: + scheduler["annotations"] = {} + if "labels" not in scheduler: + scheduler["labels"] = {} + + # Add the same pod-group annotations and labels to scheduler + scheduler["annotations"][f"{kueue_prefix}pod-group-total-count"] = str( + pod_group_total_count + ) + scheduler["labels"][f"{kueue_prefix}pod-group-name"] = pod_group_name diff --git a/projects/llm_d/orchestration/runtime_config.py b/projects/llm_d/orchestration/runtime_config.py index bc1ce1adf..dcfc9a7f1 100644 --- a/projects/llm_d/orchestration/runtime_config.py +++ b/projects/llm_d/orchestration/runtime_config.py @@ -12,7 +12,7 @@ import yaml -from projects.core.dsl.utils import slugify_identifier, truncate_k8s_name +from projects.core.dsl.utils import slugify_identifier from projects.core.library import config, env, run logger = logging.getLogger(__name__) @@ -28,9 +28,10 @@ class RunSpec: model_slug: str deployment_profile_name: str deployment_profile_slug: str + benchmark_key: str | None + benchmark_slug: str | None namespace: str artifact_dirname: str - namespace_is_managed: bool def init() -> Path: @@ -52,16 +53,7 @@ def ensure_artifact_directories(artifact_dir: Path) -> None: def _get_runtime_value(key: str) -> Any: - return config.project.get_config(f"runtime.{key}", None) - - -def _assert_no_legacy_model_key() -> None: - legacy_model_key = _get_runtime_value("model_key") - if legacy_model_key not in (None, ""): - raise ValueError( - "llm_d no longer supports runtime.model_key. " - "Use runtime.model_name with a literal Hugging Face model name instead." - ) + return config.project.get_config(f"runtime.{key}", None, warn=False) def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: @@ -99,24 +91,196 @@ def _normalize_string_or_list(value: Any, field_name: str) -> list[str]: return [value] if value else [] -def _deep_merge(base: Any, override: Any) -> Any: +def deep_merge(base: Any, override: Any) -> Any: if not isinstance(base, dict) or not isinstance(override, dict): return copy.deepcopy(override) merged = copy.deepcopy(base) for key, value in override.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = _deep_merge(merged[key], value) + merged[key] = deep_merge(merged[key], value) else: merged[key] = copy.deepcopy(value) return merged +def _extract_value_from_profile_name(profile_name: str, field_name: str) -> int: + """Extract configuration values from profile names. + + Supports patterns: + - simple-tp4-x4: tensor_parallelism=4, replicas=4 + - intelligentrouting-tp4-x4: tensor_parallelism=4, replicas=4 + - pd-d.x2-p.tp4-d.tp4-p.x1: prefill_pods=1, decode_pods=2, tensor_parallelism=4 + + Args: + profile_name: The deployment profile name + field_name: The field to extract (tensor_parallelism, replicas, prefill_pods, decode_pods, prefill_tensor_parallelism) + + Returns: + Extracted integer value + + Raises: + ValueError: If the pattern is not recognized or value cannot be extracted + """ + # P/D pattern: pd-d.x2-p.tp4-d.tp4-p.x1 + if profile_name.startswith("pd-"): + # P/D naming: pd-d.x{decode_pods}-p.tp{tensor_parallelism}-d.tp{decode_tensor_parallelism}-p.x{prefill_pods} + # Extract different components + if field_name == "tensor_parallelism": + # For main container: d.tp4 or tp4 + match = re.search(r"d\.tp(\d+)", profile_name) + if match: + return int(match.group(1)) + match = re.search(r"(? dict[str, Any]: + """Resolve FROM_NAME placeholders in profile configuration with values from profile name. + + Args: + profile_name: The deployment profile name to extract values from + profile_data: The profile configuration that may contain FROM_NAME placeholders + + Returns: + Profile configuration with FROM_NAME placeholders resolved to actual values + """ + resolved = copy.deepcopy(profile_data) + + def resolve_value(obj: Any, path: str = "") -> Any: + if isinstance(obj, dict): + result = {} + for key, value in obj.items(): + current_path = f"{path}.{key}" if path else key + result[key] = resolve_value(value, current_path) + return result + elif isinstance(obj, list): + return [resolve_value(item, path) for item in obj] + elif obj == "FROM_NAME": + # Determine which field to extract based on the path + path_parts = path.split(".") + field_name = path_parts[-1] # Get the last part of the path + + # Map field names for P/D deployments + if len(path_parts) >= 2 and path_parts[-2] in ("prefill", "decode"): + container_type = path_parts[-2] # prefill or decode + if field_name == "replicas": + field_name = f"{container_type}_pods" + elif field_name == "tensor_parallelism": + field_name = f"{container_type}_tensor_parallelism" + + try: + return _extract_value_from_profile_name(profile_name, field_name) + except ValueError as e: + logger.warning(f"Failed to extract {field_name} from {profile_name}: {e}") + return obj # Return the original FROM_NAME if extraction fails + else: + return obj + + return resolve_value(resolved) + + def _load_yaml(path: Path) -> Any: with path.open(encoding="utf-8") as handle: return yaml.safe_load(handle) +def _load_scheduler_with_epp_config( + config_dir: Path, scheduler_manifest: str, deployment_profile: dict[str, Any] +) -> dict[str, Any]: + """Load scheduler configuration using router template with EPP config replacement. + + Uses configurable paths from runtime.router config. + + Args: + config_dir: Configuration directory path + scheduler_manifest: Path to EPP config file (e.g., "manifests/deployments/approximate-prefix-cache.yaml") + deployment_profile: Resolved deployment profile for templating + + Returns: + Scheduler data with EPP config content replacing placeholder + """ + + # Get configurable paths and placeholder + router_template_file = config.project.get_config("runtime.router.template.file") + router_placeholder = config.project.get_config("runtime.router.template.placeholder") + epp_config_dir = config.project.get_config("runtime.router.epp") + + # Load the router template + router_template_path = config_dir / router_template_file + if not router_template_path.exists(): + raise FileNotFoundError(f"Router template not found at {router_template_path}") + + # Load the EPP config content + # Convert scheduler_manifest path to EPP config directory + # e.g., "manifests/deployments/approximate-prefix-cache.yaml" -> "manifests/scheduler_config/approximate-prefix-cache.yaml" + manifest_path = Path(scheduler_manifest) + epp_config_filename = manifest_path.name + epp_config_path = config_dir / epp_config_dir / epp_config_filename + + if not epp_config_path.exists(): + raise FileNotFoundError(f"EPP config not found at {epp_config_path}") + + # Load both files + router_template_data = _load_yaml(router_template_path) + epp_config_data = _load_yaml(epp_config_path) + + # Replace placeholder in the parsed data structure (avoids YAML formatting issues) + def replace_placeholder_in_structure(obj): + if isinstance(obj, dict): + return {key: replace_placeholder_in_structure(value) for key, value in obj.items()} + elif isinstance(obj, list): + return [replace_placeholder_in_structure(item) for item in obj] + elif isinstance(obj, str) and obj == router_placeholder: + # Convert EPP config to YAML string for the --config-text argument + return yaml.dump(epp_config_data, default_flow_style=False, sort_keys=False).strip() + else: + return obj + + # Perform the replacement in the structure + replaced_data = replace_placeholder_in_structure(router_template_data) + + return replaced_data + + def get_config_dir() -> Path: """Get the LLM-D configuration directory""" return ORCHESTRATION_DIR @@ -128,61 +292,25 @@ def get_job_name() -> str: if job_name: return job_name - preset_name = config.project.get_config("runtime.selected_preset") - return f"local-{preset_name}" + deployment_profile = config.project.get_config("runtime.deployment_profile") + return f"local-{deployment_profile}" def get_platform_config() -> dict[str, Any]: """Get the normalized platform configuration""" - return normalize_platform_config(copy.deepcopy(config.project.get_config("platform"))) - - -def _derive_base_namespace() -> str: - platform_data = get_platform_config() - namespace_override = _get_runtime_value("namespace_override") - namespace_config = platform_data["cluster"]["namespace"] - default_namespace = namespace_config.get("name") - - if namespace_override: - return namespace_override - if default_namespace: - return default_namespace - - return derive_namespace( - get_job_name(), - namespace_config["prefix"], - namespace_config["max_length"], + return normalize_platform_config( + copy.deepcopy(config.project.get_config("platform", print=False)) ) def get_namespace() -> str: - """Get the resolved namespace for this execution""" - return _derive_base_namespace() - - -_namespace_is_managed_override: bool | None = None - - -def get_namespace_is_managed() -> bool: - """Check if namespace is managed (auto-derived) vs explicitly configured. - - Inside activate_run_spec(), returns the base managed state captured before - the run-spec override was applied (otherwise the namespace_override set by - activate_run_spec would always flip this to False). - """ - if _namespace_is_managed_override is not None: - return _namespace_is_managed_override - - namespace_override = _get_runtime_value("namespace_override") - platform_data = get_platform_config() - default_namespace = platform_data["cluster"]["namespace"].get("name") - - return namespace_override is None and default_namespace is None + """Get the namespace for this execution""" + return _get_runtime_value("namespace") def get_model_name() -> str: """Get the selected Hugging Face model name""" - _assert_no_legacy_model_key() + model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") if len(model_names) != 1: raise ValueError( @@ -214,7 +342,7 @@ def get_served_model_name(model_name: str | None = None) -> str: def get_model_cache_config() -> dict[str, Any]: """Get the model cache configuration""" - return copy.deepcopy(config.project.get_config("model_cache")) + return copy.deepcopy(config.project.get_config("model_cache", print=False)) def get_benchmark_keys() -> list[str]: @@ -223,19 +351,25 @@ def get_benchmark_keys() -> list[str]: def _resolve_benchmark_config(benchmark_name: str) -> dict[str, Any]: benchmark = copy.deepcopy( - config.project.get_config(f"workloads.benchmarks['{benchmark_name}']") + config.project.get_config(f"workloads.benchmarks['{benchmark_name}']", print=False) ) - workload_defaults = copy.deepcopy(config.project.get_config("workloads")) + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") for key in default_keys: if key in workload_defaults and key not in benchmark: benchmark[key] = workload_defaults[key] + # Merge vllm_args from default benchmark if not present in specific benchmark + if "vllm_args" not in benchmark: + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) + if "vllm_args" in default_benchmark: + benchmark["vllm_args"] = copy.deepcopy(default_benchmark["vllm_args"]) + benchmark_args = benchmark.get("args", {}) workload_args = workload_defaults.get("args", {}) if workload_args: - benchmark["args"] = _deep_merge(workload_args, benchmark_args) + benchmark["args"] = deep_merge(workload_args, benchmark_args) return benchmark @@ -253,22 +387,48 @@ def get_benchmark_config() -> dict[str, Any] | None: return _resolve_benchmark_config(benchmark_keys[0]) -def get_benchmark_configs() -> list[tuple[str, dict[str, Any]]]: - return [ - (benchmark_key, _resolve_benchmark_config(benchmark_key)) - for benchmark_key in get_benchmark_keys() - ] +def get_workload_config() -> dict[str, Any] | None: + """Get workload configuration, falling back to default if no benchmark is specified.""" + benchmark_config = get_benchmark_config() + if benchmark_config: + return benchmark_config + # No benchmark specified, return default workload configuration + workload_defaults = copy.deepcopy(config.project.get_config("workloads", print=False)) + default_benchmark = workload_defaults.get("benchmarks", {}).get("default", {}) -def get_benchmark_job_names() -> list[str]: - """Distinct k8s benchmark job names for the active run, in order. + if not default_benchmark: + return None + + # Apply same merging logic as _resolve_benchmark_config + default_keys = ("job_name", "image", "pvc_size", "timeout_seconds") + for key in default_keys: + if key in workload_defaults and key not in default_benchmark: + default_benchmark[key] = workload_defaults[key] + + # Merge args from workloads.args to default benchmark args + benchmark_args = default_benchmark.get("args", {}) + workload_args = workload_defaults.get("args", {}) + if workload_args: + default_benchmark["args"] = deep_merge(workload_args, benchmark_args) + + return default_benchmark - A single benchmark_key collapses to one name; multiple benchmarks that - share a default job_name dedupe to one entry. Empty when benchmarking - is disabled, so cleanup paths can treat it uniformly. - """ - all_names = [benchmark.get("job_name") for _, benchmark in get_benchmark_configs()] - return list(dict.fromkeys(name for name in all_names if name)) + +def get_benchmark_deployment_overrides() -> dict[str, Any]: + """Return deployment overrides from the active benchmark config, if any.""" + benchmark = get_benchmark_config() + if benchmark is None: + return {} + return benchmark.get("deployment_overrides", {}) + + +def get_benchmark_job_name() -> str | None: + """The k8s benchmark job name for the active run, or None when benchmarking is disabled.""" + benchmark = get_benchmark_config() + if benchmark is None: + return None + return benchmark.get("job_name") def get_deployment_profile_name() -> str: @@ -284,21 +444,141 @@ def get_deployment_profile_name() -> str: return deployment_profiles[0] +def _resolve_template_profile( + profile_name: str, deployment_config: dict[str, Any] +) -> dict[str, Any]: + """Resolve a profile name to a template-based configuration if no direct match exists. + + Args: + profile_name: The deployment profile name (e.g., "simple-tp4-x4") + deployment_config: The full deployments configuration + + Returns: + Template configuration to use, or raises ValueError if no template matches + + Raises: + ValueError: If no template pattern matches the profile name + """ + templates = deployment_config.get("templates", {}) + if not templates: + raise ValueError("No templates section found in deployment config") + + # Extract first component and check if it matches a template + template_name = profile_name.partition("-")[0] + if template_name not in templates: + raise ValueError(f"No template found for profile name pattern: {profile_name}") + + logger.info(f"Using template {template_name} for profile {profile_name}") + return copy.deepcopy(templates[template_name]) + + def get_deployment_profile() -> dict[str, Any]: profile_name = get_deployment_profile_name() - deployment_config = copy.deepcopy(config.project.get_config("deployments")) - profile = copy.deepcopy(deployment_config[profile_name]) + deployment_config = copy.deepcopy(config.project.get_config("deployments", print=False)) + + # First try direct profile lookup in profiles section + profiles = deployment_config.get("profiles", {}) + if profile_name in profiles: + profile = copy.deepcopy(profiles[profile_name]) + else: + # Fall back to template-based resolution + profile = _resolve_template_profile(profile_name, deployment_config) + defaults = copy.deepcopy(deployment_config.get("defaults", {})) scheduler_manifest = profile.pop("scheduler_manifest", None) - resolved_profile = _deep_merge(defaults, profile) - if scheduler_manifest: - scheduler_data = _load_yaml(get_config_dir() / scheduler_manifest) - resolved_profile = _deep_merge(resolved_profile, scheduler_data) + resolved_profile = deep_merge(defaults, profile) + + # Resolve FROM_NAME placeholders with values extracted from profile name + resolved_profile = _resolve_from_name_values(profile_name, resolved_profile) + + scheduler_content = None + if scheduler_manifest is None: + # scheduler disabled, nothing to do + return resolved_profile + + if isinstance(scheduler_manifest, dict) and len(scheduler_manifest) == 0: # {} + # Create simplified scheduler with just basic container structure - override completely + scheduler_content = { + "scheduler": { + "template": { + "containers": [{"name": "main"}], + } + } + } + # Don't merge - directly assign to override template defaults + resolved_profile["scheduler"] = scheduler_content["scheduler"] + else: + # use __router_template__.yaml with EPP config replacement + scheduler_content = _load_scheduler_with_epp_config( + get_config_dir(), scheduler_manifest, resolved_profile + ) + + resolved_profile = deep_merge(resolved_profile, scheduler_content) + + # Configure scheduler node selector from platform config + platform_node_selector = config.project.get_config( + "platform.inference_service.scheduler.node_selector" + ) + if platform_node_selector: + scheduler_node_selector = { + "scheduler": { + "template": { + "nodeSelector": platform_node_selector, + } + } + } + resolved_profile = deep_merge(resolved_profile, scheduler_node_selector) return resolved_profile +def get_pd_config() -> dict[str, Any]: + """Get P/D configuration from active deployment profile.""" + profile = get_deployment_profile() + return profile.get("pd_config", {}) + + +def get_prefill_pod_count() -> int: + """Get number of prefill pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "prefill_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails + pd_config = get_pd_config() + return pd_config.get("prefill_pods", 1) + + +def get_decode_pod_count() -> int: + """Get number of decode pods for current deployment profile.""" + profile_name = get_deployment_profile_name() + if profile_name: + try: + return _extract_value_from_profile_name(profile_name, "decode_pods") + except ValueError: + pass + + # Fallback to pd_config if profile name extraction fails + pd_config = get_pd_config() + return pd_config.get("decode_pods", 1) + + +def get_scheduler_config() -> str: + """Get scheduler configuration for current deployment profile.""" + pd_config = get_pd_config() + return pd_config.get("scheduler_config", "default") + + +def is_pd_deployment() -> bool: + """Check if current deployment profile is a P/D deployment.""" + profile = get_deployment_profile() + return "pd_config" in profile + + def get_smoke_request() -> dict[str, Any]: """Get the smoke request configuration""" smoke_request_key = config.project.get_config("runtime.smoke_request_key") @@ -306,12 +586,15 @@ def get_smoke_request() -> dict[str, Any]: def get_run_specs() -> list[RunSpec]: - _assert_no_legacy_model_key() model_names = _normalize_string_or_list(_get_runtime_value("model_name"), "runtime.model_name") profile_names = _normalize_string_or_list( _get_runtime_value("deployment_profile"), "runtime.deployment_profile", ) + benchmark_keys = _normalize_string_or_list( + _get_runtime_value("benchmark_key"), + "runtime.benchmark_key", + ) if not model_names: raise ValueError("runtime.model_name must be set to a model name or list of model names") @@ -320,25 +603,21 @@ def get_run_specs() -> list[RunSpec]: "runtime.deployment_profile must be set to a deployment profile or list of profiles" ) - combinations = list(product(model_names, profile_names)) - base_namespace = _derive_base_namespace() - namespace_max_length = get_platform_config()["cluster"]["namespace"]["max_length"] - # Compute base managed state once (ignores per-spec overrides) - namespace_is_managed = get_namespace_is_managed() + if benchmark_keys: + benchmark_entries = [ + (key, slugify_identifier(key, max_length=24)) for key in benchmark_keys + ] + else: + benchmark_entries = [(None, None)] + + combinations = list(product(model_names, profile_names, benchmark_entries)) + namespace = get_namespace() + run_specs: list[RunSpec] = [] - for model_name, profile_name in combinations: + for model_name, profile_name, (bench_key, bench_slug) in combinations: model_slug = get_model_slug(model_name) profile_slug = slugify_identifier(profile_name, max_length=24) - if len(combinations) == 1: - namespace = base_namespace - artifact_dirname = "llmd_run" - else: - namespace = truncate_k8s_name( - f"{base_namespace}-{model_slug}-{profile_slug}", - max_length=namespace_max_length, - ) - artifact_dirname = f"llmd_run_{profile_slug}_{model_slug}" run_specs.append( RunSpec( @@ -346,9 +625,10 @@ def get_run_specs() -> list[RunSpec]: model_slug=model_slug, deployment_profile_name=profile_name, deployment_profile_slug=profile_slug, + benchmark_key=bench_key, + benchmark_slug=bench_slug, namespace=namespace, - artifact_dirname=artifact_dirname, - namespace_is_managed=namespace_is_managed, + artifact_dirname=f"llmd__{bench_key or 'default'}__{profile_slug}", ) ) @@ -358,37 +638,26 @@ def get_run_specs() -> list[RunSpec]: @contextmanager def activate_run_spec(run_spec: RunSpec): """Temporarily activate a run spec by setting its runtime config values.""" - global _namespace_is_managed_override saved = { "model_name": _get_runtime_value("model_name"), "deployment_profile": _get_runtime_value("deployment_profile"), "namespace_override": _get_runtime_value("namespace_override"), + "benchmark_key": _get_runtime_value("benchmark_key"), } - prev_managed_override = _namespace_is_managed_override config.project.set_config("runtime.model_name", run_spec.model_name) config.project.set_config("runtime.deployment_profile", run_spec.deployment_profile_name) - config.project.set_config("runtime.namespace_override", run_spec.namespace) - _namespace_is_managed_override = run_spec.namespace_is_managed + config.project.set_config("runtime.benchmark_key", run_spec.benchmark_key) try: yield finally: config.project.set_config("runtime.model_name", saved["model_name"]) config.project.set_config("runtime.deployment_profile", saved["deployment_profile"]) - config.project.set_config("runtime.namespace_override", saved["namespace_override"]) - _namespace_is_managed_override = prev_managed_override + config.project.set_config("runtime.benchmark_key", saved["benchmark_key"]) def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: - cluster = platform_data["cluster"] - if "namespace" not in cluster: - cluster["namespace"] = { - "name": cluster.pop("namespace_name", None), - "prefix": cluster.pop("namespace_prefix"), - "max_length": cluster.pop("namespace_max_length"), - } - operators = platform_data["operators"] if isinstance(operators, list): platform_data["operators"] = { @@ -401,23 +670,6 @@ def normalize_platform_config(platform_data: dict[str, Any]) -> dict[str, Any]: return platform_data -def derive_namespace(job_name: str, prefix: str, max_length: int) -> str: - slug = re.sub(r"[^a-z0-9-]+", "-", job_name.lower()) - slug = re.sub(r"-{2,}", "-", slug).strip("-") - if not slug: - slug = "run" - - if slug.startswith(f"{prefix}-"): - namespace = slug - else: - namespace = f"{prefix}-{slug}" - - namespace = namespace[:max_length].rstrip("-") - if not namespace: - raise ValueError(f"Could not derive a valid namespace from job name: {job_name}") - return namespace - - def version_tuple(value: str) -> tuple[int, ...]: numbers = re.findall(r"\d+", value) return tuple(int(number) for number in numbers[:3]) diff --git a/projects/llm_d/orchestration/test_phase.py b/projects/llm_d/orchestration/test_phase.py index f16034aad..6e8455b2b 100644 --- a/projects/llm_d/orchestration/test_phase.py +++ b/projects/llm_d/orchestration/test_phase.py @@ -1,13 +1,18 @@ from __future__ import annotations import logging +import shutil import sys from pathlib import Path from typing import Any +import yaml + +from projects.core.ci_entrypoint.prepare_ci import CI_METADATA_DIRNAME from projects.core.dsl import shell from projects.core.dsl.utils import slugify_identifier -from projects.core.library import env +from projects.core.dsl.utils.k8s import oc +from projects.core.library import config, env from projects.core.library.postprocess import run_and_postprocess, write_test_labels from projects.core.library.run import SignalInterrupt from projects.core.orchestration.utils.k8s import ensure_namespace @@ -16,6 +21,8 @@ from projects.guidellm.toolbox.run_smoke_request import main as run_smoke_request_command from projects.kserve.toolbox.capture_llmisvc_state import main as capture_llmisvc_state from projects.kserve.toolbox.deploy_llmisvc import main as deploy_llmisvc +from projects.kserve.toolbox.wait_kserve_ready import main as wait_kserve_ready +from projects.llm_d.orchestration import runtime_config from projects.llm_d.orchestration.prepare_phase import prepare_model_cache from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, @@ -26,9 +33,141 @@ logger = logging.getLogger(__name__) +def _delete_resources_by_type(resource_type: str, namespace: str, description: str) -> None: + """Delete all resources of a given type in the namespace. + + Args: + resource_type: Kubernetes resource type (e.g., 'llminferenceservice', 'workload') + namespace: Target namespace + description: Human-readable description for logging + """ + logger.info("Deleting all %s in namespace %s", description, namespace) + + result = oc( + "get", + resource_type, + "-n", + namespace, + "--no-headers", + "-o", + "name", + check=False, + ) + + if result.returncode == 0 and result.stdout.strip(): + resource_names = result.stdout.strip().split("\n") + logger.info( + "Found %d %s to delete: %s", + len(resource_names), + description, + ", ".join(resource_names), + ) + + # Delete all found resources + for resource_name in resource_names: + logger.info("Deleting %s", resource_name) + oc("delete", resource_name, "-n", namespace, check=False) + + logger.info("Successfully deleted all existing %s", description) + else: + logger.info("No existing %s found in namespace %s", description, namespace) + + +def cleanup_existing_resources(namespace: str) -> None: + """Delete all existing LLMInferenceServices and Kueue workloads if configured. + + Args: + namespace: Target namespace for cleanup + """ + delete_all_on_start = config.project.get_config("runtime.kserve.delete_all_on_start", False) + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) + + if not delete_all_on_start: + return + + if reuse_existing: + logger.info("Skipping cleanup - reuse_existing is enabled") + return + + logger.info("Starting cleanup of existing resources in namespace %s", namespace) + + try: + # Delete LLMInferenceServices + _delete_resources_by_type("llminferenceservice", namespace, "LLMInferenceServices") + + # Also delete Kueue workloads if Kueue is enabled + enable_kueue = config.project.get_config("runtime.kueue.enabled", False) + if enable_kueue: + _delete_resources_by_type("workload", namespace, "Kueue workloads") + + except Exception as e: + logger.warning("Failed to delete existing resources: %s, continuing with test", e) + + +def ensure_kueue_local_queue() -> None: + """Create LocalQueue when kueue is enabled.""" + enable_kueue = config.project.get_config("runtime.kueue.enabled") + if not enable_kueue: + return + + queue_name = config.project.get_config("runtime.kueue.queue_name") + manifest_path = config.project.get_config("runtime.kueue.local_queue_manifest") + namespace = runtime_config.get_namespace() + + logger.info("Creating LocalQueue: %s", queue_name) + + # Read and parse the YAML template + config_dir = runtime_config.get_config_dir() + template_file = config_dir / manifest_path + + with template_file.open(encoding="utf-8") as f: + local_queue_manifest = yaml.safe_load(f) + + # Update the fields + local_queue_manifest["metadata"]["name"] = queue_name + local_queue_manifest["metadata"]["namespace"] = namespace + local_queue_manifest["spec"]["clusterQueue"] = queue_name + + # Write manifest to manifests directory and apply + manifests_dir = env.ARTIFACT_DIR / "manifests" + manifests_dir.mkdir(parents=True, exist_ok=True) + manifest_file = manifests_dir / f"{queue_name}-localqueue.yaml" + + write_yaml(manifest_file, local_queue_manifest) + + # Apply the manifest + oc("apply", "-f", str(manifest_file)) + logger.info("LocalQueue %s created successfully", queue_name) + + +def extract_kpi_labels_from_config() -> dict[str, str]: + """Extract kpi_labels from project configuration. + + Returns: + Dictionary of kpi_labels for system context + """ + kpi_labels = {} + + # Extract platform name + platform_name = config.project.get_config("cpt.kpi.labels.platform_name") + if platform_name: + kpi_labels["platform"] = platform_name + + # Extract gpu_type + gpu_type = config.project.get_config("cpt.kpi.labels.gpu_type") + if gpu_type: + kpi_labels["gpu_type"] = gpu_type + + # Extract test_harness + test_harness = config.project.get_config("cpt.kpi.labels.test_harness") + if test_harness: + kpi_labels["test_harness"] = test_harness + + return kpi_labels + + def create_test_labels() -> None: """Create __test_labels__.yaml with model name and guidellm configuration.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() deployment_profile = runtime_config.get_deployment_profile_name() @@ -39,23 +178,119 @@ def create_test_labels() -> None: "deployment_profile": deployment_profile, } - # Add guidellm configuration information if benchmark_keys: - labels["guidellm_loadshape"] = ( - benchmark_keys[0] if len(benchmark_keys) == 1 else benchmark_keys - ) + labels["guidellm_loadshape"] = benchmark_keys[0] + + # Extract kpi_labels from config + kpi_labels = extract_kpi_labels_from_config() - write_test_labels(env.ARTIFACT_DIR, labels) + write_test_labels(env.ARTIFACT_DIR, labels, kpi_labels=kpi_labels if kpi_labels else None) logger.info("Created test labels: %s", labels) + # Dump config.project to config.yaml + config_path = env.ARTIFACT_DIR / "config.yaml" + try: + with config_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(config.project.config, f, sort_keys=False) + logger.info("Saved project configuration to: %s", config_path) + except Exception as e: + logger.warning("Failed to save project configuration: %s", e) + + # Copy fournos_fjob.yaml if available + fournos_source = env.ARTIFACT_DIR / CI_METADATA_DIRNAME / "fournos_fjob.yaml" + if fournos_source.exists(): + fournos_dest = env.ARTIFACT_DIR / "fournos_fjob.yaml" + try: + shutil.copy2(fournos_source, fournos_dest) + logger.info("Copied fournos job file: %s -> %s", fournos_source, fournos_dest) + except Exception as e: + logger.warning("Failed to copy fournos job file: %s", e) + else: + logger.debug("No fournos job file found at: %s", fournos_source) + + +def update_test_labels_with_status(success: bool, message: str) -> None: + """Update __test_labels__.yaml with test execution status. + + Args: + success: True if test succeeded, False if failed + message: Status message describing the result + """ + test_labels_path = env.ARTIFACT_DIR / "__test_labels__.yaml" + + # Read existing labels + if test_labels_path.exists(): + with test_labels_path.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) + else: + # Fallback if labels file doesn't exist + data = {"version": "1", "labels": {}} + + # Add completion information as separate top-level field + data["completion"] = { + "success": success, + "message": message, + } + + # Write updated labels + with test_labels_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(data, f, sort_keys=False) + + logger.info( + "Updated test labels with completion status: success=%s, message=%s", success, message + ) + + +def run_all_tests(stop_on_error: bool = False) -> int: + """Run tests for all run specifications without post-processing. + + Args: + stop_on_error: If True, stop on the first test failure + + Returns: + Maximum exit code from all tests + """ + from projects.llm_d.orchestration import runtime_config + + max_exit_code = 0 + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + with env.NextArtifactDir(run_spec.artifact_dirname): + try: + exit_code = do_test() + max_exit_code = max(max_exit_code, exit_code) + + if exit_code != 0 and stop_on_error: + logger.error( + f"Test failed with exit code {exit_code}, stopping due to stop_on_error" + ) + return exit_code + except Exception as e: + logger.exception(f"Test failed with exception: {e}") + # Note: Status update already handled by do_test() exception handler + max_exit_code = 1 + if stop_on_error: + logger.error("Stopping due to stop_on_error") + return 1 + + return max_exit_code + def run() -> int: """Main test function that wraps do_test() with outcome postprocessing.""" + + dry_run = config.project.get_config("runtime.kserve.dry_run", False) + if dry_run: + ret = do_test() + logger.info("Kserve dry-run mode enabled - Skipping caliper post-processing") + return ret + return run_and_postprocess(do_test) def run_finalizers( endpoint_url: str | None, + llmisvc_name: str | None, primary_exc: tuple[type[BaseException], BaseException, Any] | None, finalizer_exc: tuple[type[BaseException], BaseException, Any] | None, ) -> tuple[type[BaseException], BaseException, Any] | None: @@ -65,6 +300,7 @@ def _run_finalizer( **kwargs, ): try: + # with MuteStdOut(reason=f"Finalizer: {description}"): callback(**kwargs) except Exception: if primary_exc is None: @@ -73,32 +309,38 @@ def _run_finalizer( logger.exception("Ignoring %s failure after primary test failure", description) return finalizer_exc - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() capture_namespace_events = platform["artifacts"]["capture_namespace_events"] + # Only capture service state if we have the llmisvc_name + if llmisvc_name: + finalizer_exc = _run_finalizer( + "capturing inference-service state", + capture_inference_service_state, + llmisvc_name=llmisvc_name, + ) + else: + logging.warning("No llmisvc name received, cannot capture the llmisvc state") + finalizer_exc = _run_finalizer( - "capture inference-service state", - capture_inference_service_state, - ) - finalizer_exc = _run_finalizer( - "write endpoint URL", + "writing endpoint URL", write_endpoint_url, artifact_dir=env.ARTIFACT_DIR, endpoint_url=endpoint_url, ) finalizer_exc = _run_finalizer( - "capture namespace events", + "capturing namespace events", capture_namespace_events_after_test, artifact_dir=env.ARTIFACT_DIR, namespace=namespace, capture_namespace_events=capture_namespace_events, ) + finalizer_exc = _run_finalizer( - "cleanup runtime resources", + "cleaning up runtime resources", cleanup_test_resources, + llmisvc_name=llmisvc_name or "llmisvc-name-not-available", ) return primary_exc, finalizer_exc @@ -106,49 +348,79 @@ def _run_finalizer( def do_test() -> int: # Load minimal config needed for orchestration flow - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() + dry_run = config.project.get_config("runtime.kserve.dry_run", False) - # Ensure namespace exists before starting any deployments - ensure_namespace( - namespace, - labels={ - "app.kubernetes.io/managed-by": "forge", - "forge.openshift.io/project": "llm_d", - }, - ) + if not dry_run: + # Ensure namespace exists before starting any deployments + ensure_namespace( + namespace, labels=config.project.get_config("platform.cluster.namespace.labels") + ) + + # Ensure LocalQueue exists when kueue is enabled + ensure_kueue_local_queue() + + # Delete all existing resources if configured + cleanup_existing_resources(namespace) endpoint_url: str | None = None primary_exc: tuple[type[BaseException], BaseException, Any] | None = None finalizer_exc: tuple[type[BaseException], BaseException, Any] | None = None - with env.NextArtifactDir("llmd_test"): - try: - # Create test labels with actual model and profile information - create_test_labels() - - endpoint_url = deploy_inference_service() - - if not endpoint_url: - raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") - run_smoke_request(endpoint_url=endpoint_url) - - run_guidellm_benchmark(endpoint_url=endpoint_url) - except Exception: - primary_exc = sys.exc_info() - except SignalInterrupt: - primary_exc = sys.exc_info() - finally: - do_finalizers = True - if primary_exc and isinstance(primary_exc[1], SignalInterrupt): - logging.warning("Caught a SignalInterrupt, skipping the finalizers") - do_finalizers = False - - if do_finalizers: - primary_exc, finalizer_exc = run_finalizers( - endpoint_url, primary_exc, finalizer_exc - ) + actual_llmisvc_name = "llmisvc-na-not-computed" + try: + # Create test labels with actual model and profile information + create_test_labels() + + # Generate the LLMInferenceService name before deployment + # so we have it available even if deployment fails + from projects.core.dsl.utils import slugify_identifier + + platform = runtime_config.get_platform_config() + inference_service = platform["inference_service"] + base_name = inference_service["name"] + deployment_profile_name = runtime_config.get_deployment_profile_name() + # Step 1: Build manifest and get actual truncated name + initial_llmisvc_name = ( + f"{base_name}-{deployment_profile_name}" if deployment_profile_name else base_name + ) + initial_llmisvc_name = slugify_identifier(initial_llmisvc_name) + + manifest_path, actual_llmisvc_name = build_inference_service_manifest(initial_llmisvc_name) + + # Step 2: Deploy using the actual name from the manifest + endpoint_url = deploy_inference_service_from_manifest(manifest_path, actual_llmisvc_name) + + if dry_run: + logging.warning("Running in dry-run mode, skipping the rest of the test steps") + update_test_labels_with_status(True, "Dry-run completed successfully") + return 0 + + if not endpoint_url: + raise ValueError("Failed to extract the endpoint_url from the LLMISVC deployment") + run_smoke_request(endpoint_url=endpoint_url) + + run_guidellm_benchmark(endpoint_url=endpoint_url) + except Exception as e: + primary_exc = sys.exc_info() + update_test_labels_with_status(False, f"Test failed with exception: {str(e)}") + except SignalInterrupt as e: + primary_exc = sys.exc_info() + update_test_labels_with_status(False, f"Test interrupted: {str(e)}") + finally: + do_finalizers = config.project.get_config("runtime.run_test_finalizers") + if primary_exc and isinstance(primary_exc[1], SignalInterrupt): + logging.warning("Caught a SignalInterrupt, skipping the finalizers") + do_finalizers = False + + if dry_run: + do_finalizers = False + + if do_finalizers: + primary_exc, finalizer_exc = run_finalizers( + endpoint_url, actual_llmisvc_name, primary_exc, finalizer_exc + ) if primary_exc is not None: raise primary_exc[1].with_traceback(primary_exc[2]) @@ -156,45 +428,201 @@ def do_test() -> int: if finalizer_exc is not None: raise finalizer_exc[1].with_traceback(finalizer_exc[2]) + # Update test labels with success status + update_test_labels_with_status(True, "Test completed successfully") + return 0 -def deploy_inference_service() -> str: - """Deploy LLMInferenceService and return endpoint URL. +def _try_reuse_existing_service( + namespace: str, llmisvc_name: str, gateway: dict[str, str], reuse_existing: bool, dry_run: bool +) -> str | None: + """Try to reuse an existing LLMInferenceService if enabled and available. + + Args: + namespace: Target namespace + llmisvc_name: Name of the LLMInferenceService + gateway: Gateway configuration + reuse_existing: Whether to attempt reuse + dry_run: Whether in dry-run mode Returns: - Gateway endpoint URL for the deployed service + Endpoint URL if reuse successful, None otherwise """ - logger.info("Starting LLMInferenceService deployment") + # Check if we should reuse existing LLMInferenceService + if reuse_existing and not dry_run: + logger.info(f"Checking if LLMInferenceService {llmisvc_name} already exists") - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config + # Check if the service already exists + try: + existing_llmisvc = oc( + "get", + "llminferenceservice", + llmisvc_name, + "-n", + namespace, + check=False, + ) + + if existing_llmisvc.returncode == 0: + logger.info( + f"Found existing LLMInferenceService {llmisvc_name}, attempting to reuse" + ) + + # Import and use the toolbox function to extract URL + # Check if the existing service has a scheduler to determine gateway address + from projects.core.dsl.utils.k8s import oc_get_json + from projects.kserve.toolbox.deploy_llmisvc import try_resolve_endpoint_url + + existing_service = oc_get_json( + "llminferenceservice", name=llmisvc_name, namespace=namespace + ) + has_scheduler = ( + existing_service.get("spec", {}).get("router", {}).get("scheduler") is not None + ) + + # Use None for status_address_name when existing service has no scheduler + gateway_status_address_name = ( + gateway["status_address_name"] if has_scheduler else None + ) + + endpoint_url = try_resolve_endpoint_url( + namespace=namespace, + inference_service_name=llmisvc_name, + gateway_status_address_name=gateway_status_address_name, + ) + + if endpoint_url: + logger.info(f"Successfully reused existing LLMInferenceService: {endpoint_url}") + return endpoint_url + else: + logger.warning( + "Existing LLMInferenceService found but no endpoint URL could be resolved, proceeding with new deployment" + ) + else: + logger.info( + f"LLMInferenceService {llmisvc_name} does not exist, proceeding with new deployment" + ) + + except Exception as e: + logger.warning( + f"Error checking for existing LLMInferenceService: {e}, proceeding with new deployment" + ) + + return None + + +def build_inference_service_manifest(llmisvc_name: str) -> tuple[Path, str]: + """Build inference service manifest and return path and actual service name. + + Args: + llmisvc_name: The initial name for the LLMInferenceService + + Returns: + Tuple of (manifest_path, actual_llmisvc_name) where actual_llmisvc_name may be truncated + """ + # Step 1: Build and write inference service manifest + manifest_path = _build_inference_service_manifest() + + # Step 2: Extract the actual deployed name from the manifest (may be truncated) + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + # Get the actual deployed name from the manifest (may be truncated) + actual_llmisvc_name = manifest["metadata"]["name"] + + logger.info( + "Built LLMInferenceService manifest: %s (actual name: %s)", + manifest_path, + actual_llmisvc_name, + ) + return manifest_path, actual_llmisvc_name + + +def deploy_inference_service_from_manifest(manifest_path: Path, actual_llmisvc_name: str) -> str: + """Deploy LLMInferenceService from pre-built manifest and return endpoint URL. + Args: + manifest_path: Path to the pre-built manifest + actual_llmisvc_name: The actual name of the LLMInferenceService from the manifest + + Returns: + Gateway endpoint URL + """ + logger.info("Starting LLMInferenceService deployment from manifest") + + # Load config where it's consumed namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() gateway = platform["gateway"] - # Step 1: Ensure model cache is ready - _prepare_model_cache() + dry_run = config.project.get_config("runtime.kserve.dry_run") + wait_readiness = config.project.get_config("runtime.kserve.wait_readiness") + reuse_existing = config.project.get_config("runtime.kserve.reuse_existing", False) - # Step 2: Build and write inference service manifest - manifest_path = _build_inference_service_manifest() + # Try to reuse existing LLMInferenceService if enabled + endpoint_url = _try_reuse_existing_service( + namespace=namespace, + llmisvc_name=actual_llmisvc_name, + gateway=gateway, + reuse_existing=reuse_existing, + dry_run=dry_run, + ) + if endpoint_url: + return endpoint_url + + # Step 1: Ensure model cache is ready (skip in dry-run) + if not dry_run: + _prepare_model_cache() + else: + logger.info("Skipping model cache preparation - dry-run mode enabled") + + # Step 2: Wait for the serving control plane to settle before creating the service. + if not dry_run and wait_readiness: + rhoai_namespace = platform["rhoai"]["namespace"] + wait_kserve_ready.run(namespace=rhoai_namespace) + + # Step 3: Check manifest for scheduler to determine gateway status address name + import yaml + + with manifest_path.open(encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + has_scheduler = ( + "spec" in manifest + and "router" in manifest["spec"] + and "scheduler" in manifest["spec"]["router"] + ) - # Step 3: Deploy the service and wait for endpoint + # Use None for status_address_name when deploying without a scheduler + gateway_status_address_name = gateway["status_address_name"] if has_scheduler else None + + # Step 4: Deploy the service and wait for endpoint logger.info("Deploying LLMInferenceService from manifest: %s", manifest_path) + + # Get scheduling wait configuration + wait_pods_scheduled = config.project.get_config("runtime.kserve.wait_long_scheduling") + endpoint_url = deploy_llmisvc.run( namespace=namespace, inference_service_manifest_path=str(manifest_path), - gateway_status_address_name=gateway["status_address_name"], + gateway_status_address_name=gateway_status_address_name, + dry_run=dry_run, + wait_pods_scheduled=wait_pods_scheduled, ) + if dry_run: + logger.info("Dry-run completed: LLMInferenceService manifest prepared: %s", manifest_path) + return str(manifest_path) + logger.info("LLMInferenceService deployed successfully, endpoint: %s", endpoint_url) return endpoint_url def _prepare_model_cache() -> None: """Ensure model cache PVC is ready for deployment.""" - from projects.llm_d.orchestration import runtime_config model_name = runtime_config.get_model_name() logger.info("Preparing model cache for model: %s", model_name) @@ -206,7 +634,6 @@ def _prepare_model_cache() -> None: def _build_inference_service_manifest() -> Path: """Build and write the LLMInferenceService manifest.""" - from projects.llm_d.orchestration import runtime_config config_dir = runtime_config.get_config_dir() namespace = runtime_config.get_namespace() @@ -216,8 +643,14 @@ def _build_inference_service_manifest() -> Path: model_slug = runtime_config.get_model_slug(model_name) deployment_profile = runtime_config.get_deployment_profile() model_cache = runtime_config.get_model_cache_config() + workload = runtime_config.get_workload_config() # Get workload config with vllm_args + + benchmark_overrides = runtime_config.get_benchmark_deployment_overrides() + if benchmark_overrides: + deployment_profile = runtime_config.deep_merge(deployment_profile, benchmark_overrides) # Build the InferenceService manifest + deployment_profile_name = runtime_config.get_deployment_profile_name() manifest = render_inference_service_from_parts( config_dir=config_dir, namespace=namespace, @@ -226,6 +659,8 @@ def _build_inference_service_manifest() -> Path: model_slug=model_slug, deployment_profile=deployment_profile, model_cache=model_cache, + deployment_profile_name=deployment_profile_name, + workload=workload, ) # Write the manifest to artifacts @@ -240,7 +675,6 @@ def _build_inference_service_manifest() -> Path: def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() @@ -262,42 +696,35 @@ def run_smoke_request(*, endpoint_url: str) -> dict[str, object]: def run_guidellm_benchmark(*, endpoint_url: str) -> None: - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config - namespace = runtime_config.get_namespace() - benchmark_configs = runtime_config.get_benchmark_configs() - - if not benchmark_configs: - return # Skip if benchmark is disabled - - for benchmark_key, benchmark in benchmark_configs: - guidellm_args = build_guidellm_args(benchmark) - if not any(arg.startswith("--processor=") for arg in guidellm_args): - guidellm_args.append(f"--processor={runtime_config.get_model_name()}") - artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" - with env.NextArtifactDir(artifact_name): - run_guidellm_benchmark_command.run( - endpoint_url=endpoint_url, - name=benchmark.get("job_name"), - namespace=namespace, - image=benchmark.get("image"), - timeout=benchmark.get("timeout_seconds"), - pvc_size=benchmark.get("pvc_size"), - guidellm_args=guidellm_args, - ) + benchmark = runtime_config.get_benchmark_config() + if benchmark is None: + return + + benchmark_key = runtime_config.get_benchmark_keys()[0] + guidellm_args = build_guidellm_args(benchmark) + if not any(arg.startswith("--processor=") for arg in guidellm_args): + guidellm_args.append(f"--processor={runtime_config.get_model_name()}") + artifact_name = f"benchmark_{slugify_identifier(benchmark_key, max_length=48)}" + with env.NextArtifactDir(artifact_name): + run_guidellm_benchmark_command.run( + endpoint_url=endpoint_url, + name=benchmark.get("job_name"), + namespace=namespace, + image=benchmark.get("image"), + timeout=benchmark.get("timeout_seconds"), + pvc_size=benchmark.get("pvc_size"), + guidellm_args=guidellm_args, + ) -def capture_inference_service_state() -> None: - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config +def capture_inference_service_state(llmisvc_name: str) -> None: + """Capture inference service state for the given llmisvc name.""" namespace = runtime_config.get_namespace() - platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] capture_llmisvc_state.run( - llmisvc_name=inference_service["name"], + llmisvc_name=llmisvc_name, namespace=namespace, ) @@ -311,28 +738,33 @@ def write_endpoint_url(*, artifact_dir: Path, endpoint_url: str | None) -> None: endpoint_file.write_text(f"{endpoint_url}\n", encoding="utf-8") -def cleanup_test_resources() -> None: - """Cleanup test resources using the toolbox script""" - # Load config where it's consumed - from projects.llm_d.orchestration import runtime_config +def cleanup_test_resources(llmisvc_name: str | None) -> None: + """Cleanup test resources using the toolbox script + + Args: + llmisvc_name: The actual LLMInferenceService name that was deployed, or None if deployment failed + """ + + # Skip cleanup when in dry-run mode + dry_run = config.project.get_config("runtime.kserve.dry_run", False) + if dry_run: + logger.info("Skipping cleanup_test_resources - dry-run mode enabled") + return + + if not llmisvc_name: + logger.warning("No LLMInferenceService name provided, cleanup may be incomplete") + return namespace = runtime_config.get_namespace() platform = runtime_config.get_platform_config() - inference_service = platform["inference_service"] smoke = platform["smoke"] - # Narrow cleanup (no broad sweep): cover every benchmark job name. Benchmarks - # typically share a single job_name, so this dedupes to one call; when - # benchmarking is disabled, run once with None so the IS/smoke cleanup still - # runs. oc delete is idempotent across iterations. - benchmark_job_names = runtime_config.get_benchmark_job_names() or [None] - for benchmark_job_name in benchmark_job_names: - cleanup_test_resources_command.run( - namespace=namespace, - inference_service_name=inference_service["name"], - smoke_pod_name=smoke["pod_name"], - benchmark_job_name=benchmark_job_name, - ) + cleanup_test_resources_command.run( + namespace=namespace, + inference_service_name=llmisvc_name, + smoke_pod_name=smoke["pod_name"], + benchmark_job_name=runtime_config.get_benchmark_job_name(), + ) def capture_namespace_events_after_test( diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..42aa4b492 --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-intelligentrouting-tp4-x4/llmisvc.yaml @@ -0,0 +1,64 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-intelligentr-4e3b09 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: intelligentrouting-tp4-x4 + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b + router: + scheduler: + template: + containers: + - name: main + nodeSelector: + nvidia.com/gpu.present: 'true' + route: {} + gateway: {} + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml new file mode 100644 index 000000000..a12df366c --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-pd-d.x2-p.tp1-d.tp4-p.x8/llmisvc.yaml @@ -0,0 +1,95 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-pd-d-x2-p-tp-bfb506 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: pd-d.x2-p.tp1-d.tp4-p.x8 + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 2 + model: + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b + router: + scheduler: + template: + containers: + - name: main + route: {} + gateway: {} + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + rdma/ib: '1' + limits: + nvidia.com/gpu: '4' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: decode + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: prefill + topologyKey: kubernetes.io/hostname + prefill: + replicas: 8 + parallelism: + tensor: 1 + template: + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '1' + rdma/ib: '1' + limits: + nvidia.com/gpu: '1' + rdma/ib: '1' + env: + - name: VLLM_ADDITIONAL_ARGS + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 --block-size 128 --kv-transfer-config + '{"kv_connector":"NixlConnector", "kv_role":"kv_both"}' + - name: VLLM_NIXL_SIDE_CHANNEL_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + metadata: + labels: + app.kubernetes.io/component: prefill + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: decode + topologyKey: kubernetes.io/hostname + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml new file mode 100644 index 000000000..a9e9aeb1b --- /dev/null +++ b/projects/llm_d/tests/reference_deployments/cpt-reference-flavors/deployment-simple-tp4-x4/llmisvc.yaml @@ -0,0 +1,55 @@ +apiVersion: serving.kserve.io/v1alpha1 +kind: LLMInferenceService +metadata: + name: llm-d-simple-tp4-x4 + namespace: forge-llm-d + annotations: + security.opendatahub.io/enable-auth: 'false' + prometheus.io/path: /metrics + prometheus.io/port: '8000' + forge.openshift.io/deployment-profile: simple-tp4-x4 + labels: + app.kubernetes.io/managed-by: forge + forge.openshift.io/project: llm_d +spec: + replicas: 4 + model: + uri: pvc://llm-d-model-openai-gpt-oss-120b-7ab79ddecd/model + name: openai-gpt-oss-120b + template: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: main + resources: + requests: + nvidia.com/gpu: '4' + limits: + nvidia.com/gpu: '4' + livenessProbe: + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 900 + periodSeconds: 60 + timeoutSeconds: 60 + failureThreshold: 1000 + readinessProbe: + failureThreshold: 10000 + httpGet: + path: /health + port: 8000 + scheme: HTTPS + initialDelaySeconds: 60 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 30 + env: + - name: VLLM_ADDITIONAL_ARGS + value: --gpu-memory-utilization=0.92 --trust-remote-code --no-enable-log-requests + --enable-prefix-caching --max-model-len=4096 + parallelism: + tensor: 4 diff --git a/projects/llm_d/tests/test_deployment_profiles.py b/projects/llm_d/tests/test_deployment_profiles.py new file mode 100644 index 000000000..f23a2731e --- /dev/null +++ b/projects/llm_d/tests/test_deployment_profiles.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +from projects.core.library import config as core_config +from projects.core.library import env + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" +REFERENCE_DIR = Path(__file__).resolve().parent / "reference_deployments" + +# Deployment presets to test +DEPLOYMENT_PRESETS = [ + "cpt-reference-flavors", +] + +# Check for save deployments mode via environment variable +SAVE_DEPLOYMENTS = os.environ.get("SAVE_DEPLOYMENTS", "false").lower() in ("true", "1", "yes") + + +@pytest.fixture(autouse=True) +def _reset_project_config(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ARTIFACT_DIR", str(tmp_path / "artifacts")) + # Unset KUBECONFIG as requested + monkeypatch.delenv("KUBECONFIG", raising=False) + env.init() + core_config.project = None + yield + core_config.project = None + + +@pytest.mark.parametrize("preset", DEPLOYMENT_PRESETS) +def test_deployment_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Test that deployment presets generate the expected LLMISVC manifests.""" + _test_preset_generates_expected_llmisvc(preset, tmp_path) + + +def _test_preset_generates_expected_llmisvc(preset: str, tmp_path: Path): + """Helper function to test a preset generates the expected LLMISVC manifest.""" + # Set up environment + artifact_dir = tmp_path / "artifacts" / preset + artifact_dir.mkdir(parents=True, exist_ok=True) + + # Create variable overrides to ensure dry-run mode + variable_overrides_path = artifact_dir / "000__ci_metadata" / "variable_overrides.yaml" + variable_overrides_path.parent.mkdir(parents=True, exist_ok=True) + variable_overrides_path.write_text( + yaml.safe_dump( + { + "runtime.kserve.dry_run": True, + "caliper.postprocess.enabled": False, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + env_vars = os.environ.copy() + env_vars["ARTIFACT_DIR"] = str(artifact_dir) + env_vars.pop("KUBECONFIG", None) + + # Run the CI script with the preset + ci_script = PROJECT_ROOT / "projects" / "llm_d" / "orchestration" / "ci.py" + cmd = [str(ci_script), "--preset", preset, "test"] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + env=env_vars, + timeout=60, # 60-second timeout + ) + except subprocess.TimeoutExpired as e: + # Show stdout when timeout occurs + stdout_output = e.stdout.decode() if e.stdout else "No stdout available" + stderr_output = e.stderr.decode() if e.stderr else "No stderr available" + pytest.fail( + f"CI command timed out after 60 seconds for preset {preset}:\n" + f"Command: {' '.join(cmd)}\n\n" + f"STDOUT (before timeout):\n{stdout_output}\n\n" + f"STDERR (before timeout):\n{stderr_output}\n\n" + ) + + # Check that the command succeeded + if result.returncode != 0: + pytest.fail( + f"CI command failed for preset {preset}:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + # Find all generated LLMISVC manifests + deployments = _find_generated_llmisvcs(artifact_dir) + + if not deployments: + pytest.fail( + f"No generated LLMISVC manifests found in {artifact_dir}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + # If saving deployments, save them and skip comparison + if SAVE_DEPLOYMENTS: + _save_deployments(preset, deployments) + pytest.skip(f"Saved {len(deployments)} deployments for preset {preset}") + return + + # Test each deployment against its reference + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + print(f"Testing deployment for profile: {profile_name}") + + if not generated_manifest.exists(): + pytest.fail( + f"Generated LLMISVC manifest not found at {generated_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + # Load the generated manifest + with generated_manifest.open("r", encoding="utf-8") as f: + generated_content = yaml.safe_load(f) + + # Determine reference manifest path + # Look in preset-specific directory structure: preset/deployment-{profile_name}/llmisvc.yaml + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + if not reference_manifest.exists(): + pytest.fail( + f"Reference manifest not found at {reference_manifest}\n\n" + f"💡 To save generated deployments as references, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + with reference_manifest.open("r", encoding="utf-8") as f: + reference_content = yaml.safe_load(f) + + # Compare the manifests + if generated_content != reference_content: + # Generate a diff for better error reporting + import difflib + + generated_yaml = yaml.dump(generated_content, default_flow_style=False, sort_keys=True) + reference_yaml = yaml.dump(reference_content, default_flow_style=False, sort_keys=True) + + diff = list( + difflib.unified_diff( + reference_yaml.splitlines(keepends=True), + generated_yaml.splitlines(keepends=True), + fromfile=f"reference/{preset}/deployment-{profile_name}/llmisvc.yaml", + tofile=f"generated/{preset}/{profile_name}/llmisvc.yaml", + lineterm="", + ) + ) + + pytest.fail( + f"Generated LLMISVC does not match reference for profile {profile_name} in preset {preset}:\n{''.join(diff)}\n\n" + f"💡 To update reference deployments with new output, use:\n" + f" SAVE_DEPLOYMENTS=true python -m pytest {__name__} -v -s" + ) + + +def _save_deployments(preset: str, deployments: list[dict[str, Path]]): + """Save generated deployments as reference manifests.""" + print(f"\nSaving {len(deployments)} deployments for preset {preset}:") + + for deployment in deployments: + profile_name = deployment["profile_name"] + generated_manifest = deployment["manifest_path"] + + # Create reference directory for this preset and deployment profile + reference_preset_dir = REFERENCE_DIR / preset + reference_deployment_dir = reference_preset_dir / f"deployment-{profile_name}" + reference_deployment_dir.mkdir(parents=True, exist_ok=True) + + # Copy the generated manifest to the reference location + reference_manifest = reference_deployment_dir / "llmisvc.yaml" + + with generated_manifest.open("r", encoding="utf-8") as f: + content = f.read() + + with reference_manifest.open("w", encoding="utf-8") as f: + f.write(content) + + print(f" Saved {profile_name} → {reference_manifest}") + + print(f"Reference deployments saved to: {reference_preset_dir}") + + +def _find_generated_llmisvcs(artifact_dir: Path) -> list[dict[str, Path]]: + """Find all generated LLMISVC manifests in the artifact directory. + + Returns: + List of dictionaries with 'profile_name' and 'manifest_path' keys + """ + # Look for the pattern: $ARTIFACT_DIR/**/deploy_llmisvc/src/llminferenceservice.yaml + deploy_llmisvc_dirs = list(artifact_dir.glob("**/*__deploy_llmisvc")) + + if not deploy_llmisvc_dirs: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") + + deployments = [] + for deploy_dir in deploy_llmisvc_dirs: + llmisvc_path = deploy_dir / "src" / "llminferenceservice.yaml" + if llmisvc_path.exists(): + # Read the manifest and extract profile name from annotations + with llmisvc_path.open("r", encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + ANNOTATION = "forge.openshift.io/deployment-profile" + profile_name = manifest.get("metadata", {}).get("annotations", {}).get(ANNOTATION) + + if not profile_name: + raise ValueError(f"Annotation '{ANNOTATION}' missing in {llmisvc_path}") + + deployments.append({"profile_name": profile_name, "manifest_path": llmisvc_path}) + + return deployments + + +def _find_generated_llmisvc(artifact_dir: Path) -> Path: + """Find the generated LLMISVC manifest in the artifact directory.""" + deployments = _find_generated_llmisvcs(artifact_dir) + + if len(deployments) == 1: + return deployments[0]["manifest_path"] + elif len(deployments) > 1: + raise ValueError( + f"Multiple deploy_llmisvc directories found in {artifact_dir}. " + f"Use _find_generated_llmisvcs() instead for multi-deployment presets." + ) + else: + raise FileNotFoundError(f"No deploy_llmisvc directory found in {artifact_dir}") diff --git a/projects/llm_d/tests/test_pr_args.py b/projects/llm_d/tests/test_pr_args.py index 11ae6a13d..9545a9e30 100644 --- a/projects/llm_d/tests/test_pr_args.py +++ b/projects/llm_d/tests/test_pr_args.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from projects.llm_d.orchestration import pr_args @@ -22,3 +24,35 @@ def test_parse_project_directives_ignores_other_projects() -> None: assert overrides == {} assert directives == [] + + +def test_rhoai_rc_image_sets_custom_catalog_config() -> None: + image = "quay.io/rhoai/rhoai-fbc-fragment@sha256:e213152dc5bdaaf5724d269484914f3e1d7b4a4959d4405cb3eca9dd6297b310" + overrides, directives = pr_args.parse_project_directives(f"/rhoai.rc-image {image}") + + assert overrides == { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + assert len(directives) == 1 + + +def test_rhoai_rc_image_without_image_raises() -> None: + with pytest.raises(ValueError, match="image cannot be empty"): + pr_args.parse_project_directives("/rhoai.rc-image") + + +def test_rhoai_rc_image_with_other_directives() -> None: + comment = """/test fournos llm_d smoke +/rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123""" + + overrides, directives = pr_args.parse_project_directives(comment) + + assert overrides["platform.rhoai.custom_catalog.enabled"] is True + assert ( + overrides["platform.rhoai.custom_catalog.image"] + == "quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + assert overrides["platform.operators.rhods-operator.channel"] == "beta" + assert len(directives) == 1 diff --git a/projects/llm_d/tests/test_profiles.py b/projects/llm_d/tests/test_profiles.py index a798fb8e2..1efce2f07 100644 --- a/projects/llm_d/tests/test_profiles.py +++ b/projects/llm_d/tests/test_profiles.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import json from pathlib import Path import pytest @@ -12,6 +14,7 @@ from projects.llm_d.orchestration.render_inference_service import ( render_inference_service_from_parts, ) +from projects.rhoai.library import deploy as rhoai_deploy PROJECT_ORCHESTRATION_DIR = Path(__file__).resolve().parents[1] / "orchestration" @@ -55,13 +58,6 @@ def test_release_deployment_profiles_have_expected_shape() -> None: for profile in (approximate, precise, distributed): assert profile["replicas"] == 1 assert profile["tensor_parallelism"] == 1 - assert profile["vllm_args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - ] assert isinstance(approximate["scheduler"], dict) assert isinstance(precise["scheduler"], dict) @@ -162,10 +158,28 @@ def test_release_preset_expands_benchmark_list_and_merges_workload_args() -> Non "multi-turn", ] - benchmark_configs = dict(runtime_config.get_benchmark_configs()) - assert benchmark_configs["concurrent-1k-1k"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["heavy-heterogeneous"]["args"]["request_type"] == "text_completions" - assert benchmark_configs["multi-turn"]["args"]["request_type"] == "text_completions" + run_specs = runtime_config.get_run_specs() + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + benchmark = runtime_config.get_benchmark_config() + assert benchmark["args"]["request_type"] == "text_completions" + + +def test_release_preset_produces_3_run_specs() -> None: + _init_project_config() + + core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 3 + assert [spec.benchmark_key for spec in run_specs] == [ + "concurrent-1k-1k", + "heavy-heterogeneous", + "multi-turn", + ] + assert all(spec.model_name == "openai/gpt-oss-120b" for spec in run_specs) + assert all(spec.deployment_profile_name == "distributed-default" for spec in run_specs) def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> None: @@ -189,13 +203,325 @@ def test_ci_init_uses_framework_project_args_preset_and_keeps_var_overrides() -> assert runtime_config.get_benchmark_keys() == ["multi-turn"] -def test_ci_init_uses_project_default_preset_when_no_explicit_preset_is_provided() -> None: - llmd_ci.init() +def test_list_vaults_only_includes_rhoai_custom_catalog_vaults_for_custom_catalog_runs() -> None: + _init_project_config() - assert runtime_config.get_model_name() == "Qwen/Qwen3-0.6B" - assert runtime_config.get_deployment_profile_name() == "approximate-prefix-cache" - # Default preset "smoke" enables the short benchmark - assert runtime_config.get_benchmark_keys() == ["short"] + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + assert "psap-rhoai-rc" not in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" not in llmd_ci.list_vaults() + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + assert "psap-rhoai-rc" in llmd_ci.list_vaults() + assert "psap-forge-staging-image-pull" in llmd_ci.list_vaults() + + +def test_prepare_phase_adds_rhoai_custom_catalog_vaults_only_for_custom_catalog_runs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + calls: list[dict[str, object]] = [] + + def _fake_init(**kwargs): + calls.append(kwargs) + + monkeypatch.setattr(llmd_ci.vault, "init", _fake_init) + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", False) + llmd_ci.init_vaults_for_phase("prepare") + + core_config.project.set_config("platform.rhoai.custom_catalog.enabled", True) + llmd_ci.init_vaults_for_phase("prepare") + + assert "psap-rhoai-rc" not in calls[0]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" not in calls[0]["mandatory_vaults"] + assert "psap-rhoai-rc" in calls[1]["mandatory_vaults"] + assert "psap-forge-staging-image-pull" in calls[1]["mandatory_vaults"] + + +def test_prepare_rhoai_operator_runs_registry_setup_before_custom_catalog_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _init_project_config() + + platform = { + "operators": { + "rhcl-operator": { + "package": "rhcl-operator", + "namespace": "openshift-operators", + "source": "redhat-operators", + "channel": "stable", + }, + "rhods-operator": { + "package": "rhods-operator", + "namespace": "redhat-ods-operator", + "source": "redhat-operators", + "channel": "stable-3.x", + }, + }, + "rhoai": { + "custom_catalog": { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, + }, + "namespace": "redhat-ods-applications", + "datasciencecluster_name": "default-dsc", + "components": ["kserve"], + "required_crds_before_dsc": ["datascienceclusters.datasciencecluster.opendatahub.io"], + "required_crds_after_dsc": ["llminferenceservices.serving.kserve.io"], + }, + } + + calls: list[object] = [] + + monkeypatch.setattr( + rhoai_deploy, "prepare_rhcl_operator", lambda platform: calls.append("rhcl") + ) + monkeypatch.setattr( + rhoai_deploy, + "prepare_rhoai_pull_secret", + lambda custom_catalog: calls.append(("pull", custom_catalog.image)), + ) + monkeypatch.setattr( + rhoai_deploy, + "deploy_rhoai_custom_catalog", + lambda custom_catalog: calls.append(("catalog", custom_catalog.publisher)), + ) + monkeypatch.setattr( + rhoai_deploy, + "ensure_operator_subscription", + lambda operator_spec: calls.append(("sub", operator_spec["package"])), + ) + monkeypatch.setattr( + rhoai_deploy, + "ensure_required_crds_before_dsc", + lambda rhoai: calls.append("crds"), + ) + + rhoai_deploy.prepare_rhoai_operator( + platform=platform, + rhoai=platform["rhoai"], + icsp_applier=lambda: calls.append("icsp"), + ) + + assert calls == [ + "rhcl", + ("pull", "quay.io/rhoai/rhoai-fbc-fragment@sha256:test"), + "icsp", + ("catalog", "RHOAI Development Catalog"), + ("sub", "rhods-operator"), + "crds", + ] + + +def test_custom_catalog_pull_secret_path_uses_explicit_vault_content( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, str] = {} + + def _fake_get_vault_content_path(vault_name: str, content_name: str): + captured["vault_name"] = vault_name + captured["content_name"] = content_name + return Path("/tmp/rhoai_rc.secret") + + monkeypatch.setattr(rhoai_deploy.vault, "get_vault_content_path", _fake_get_vault_content_path) + + path = rhoai_deploy.custom_catalog_pull_secret_path( + { + "pull_secret": { + "vault": { + "name": "psap-rhoai-rc", + "content": "rhoai_rc.secret", + } + }, + } + ) + + assert path == Path("/tmp/rhoai_rc.secret") + assert captured == { + "vault_name": "psap-rhoai-rc", + "content_name": "rhoai_rc.secret", + } + + +def test_wait_for_rhoai_pull_secret_ready_treats_empty_mcp_status_as_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + mcp_outputs = iter(["", "True"]) + clock = {"value": 0.0} + + def _fake_monotonic() -> float: + value = clock["value"] + clock["value"] += 0.1 + return value + + def _fake_sleep(_seconds: float) -> None: + clock["value"] += 0.1 + + def _fake_oc_get_json(*args, **kwargs): + payload = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + return { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps(payload).encode("utf-8")).decode( + "utf-8" + ) + } + } + + def _fake_oc(*args, **kwargs): + if args[:2] == ("get", "mcp"): + return type("Result", (), {"stdout": next(mcp_outputs), "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc_get_json", _fake_oc_get_json) + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy.time, "monotonic", _fake_monotonic) + monkeypatch.setattr(rhoai_deploy.time, "sleep", _fake_sleep) + + rhoai_deploy.wait_for_rhoai_pull_secret_ready(timeout_seconds=1, poll_interval_seconds=0) + + +def test_registries_present_accepts_parent_registry_auth_entries() -> None: + pull_secret = { + "auths": { + "quay.io": {"auth": "token-a"}, + "registry.stage.redhat.io": {"auth": "token-b"}, + } + } + + assert rhoai_deploy._registries_present( + pull_secret, + ( + "quay.io/rhoai", + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", + ), + ) + + +def test_prepare_rhoai_pull_secret_merges_dockerconfigjson_registry_auths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _init_project_config() + + rc_pull_secret_path = tmp_path / "rhoai_rc.secret" + rc_pull_secret_path.write_text( + "rhoai+rhoai_external_readonly_bot:token-c", + encoding="utf-8", + ) + + staging_pull_secret_path = tmp_path / ".dockerconfigjson" + staging_pull_secret_path.write_text( + json.dumps( + { + "auths": { + "registry.stage.redhat.io/rhaii": {"auth": "token-a"}, + "registry.stage.redhat.io/rhaii-early-access": {"auth": "token-b"}, + } + } + ), + encoding="utf-8", + ) + + current_secret = { + "data": { + ".dockerconfigjson": base64.b64encode(json.dumps({"auths": {}}).encode("utf-8")).decode( + "utf-8" + ) + } + } + merged_payload: dict[str, object] = {} + + monkeypatch.setattr( + rhoai_deploy.vault, + "get_vault_content_path", + lambda vault_name, content_name: ( + rc_pull_secret_path + if (vault_name, content_name) == ("psap-rhoai-rc", "rhoai_rc.secret") + else staging_pull_secret_path + ), + ) + monkeypatch.setattr(rhoai_deploy, "oc_get_json", lambda *args, **kwargs: current_secret) + + def _fake_oc(*args, **kwargs): + if args[:2] == ("set", "data"): + from_file = next((arg for arg in args if str(arg).startswith("--from-file=")), "") + payload_path = Path(from_file.rsplit("=", 1)[1]) + merged_payload.update(json.loads(payload_path.read_text(encoding="utf-8"))) + return type("Result", (), {"stdout": "", "returncode": 0})() + if args[:2] == ("registry", "login"): + registry = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--registry=") + ) + auth_basic = next( + arg.split("=", 1)[1] for arg in args if str(arg).startswith("--auth-basic=") + ) + temp_path = Path( + next( + arg + for arg in args + if not str(arg).startswith("--") and arg != "registry" and arg != "login" + ) + ) + current = json.loads(temp_path.read_text(encoding="utf-8")) + current.setdefault("auths", {})[registry] = {"auth": auth_basic} + temp_path.write_text(json.dumps(current, indent=2, sort_keys=True), encoding="utf-8") + return type("Result", (), {"stdout": "", "returncode": 0})() + raise AssertionError(f"Unexpected oc call: {args}") + + monkeypatch.setattr(rhoai_deploy, "oc", _fake_oc) + monkeypatch.setattr(rhoai_deploy, "wait_for_rhoai_pull_secret_ready", lambda **kwargs: None) + + rhoai_deploy.prepare_rhoai_pull_secret( + rhoai_deploy.RhoaiCustomCatalogConfig.model_validate( + { + "enabled": True, + "name": "rhoai-catalog-dev", + "namespace": "openshift-marketplace", + "image": "quay.io/rhoai/rhoai-fbc-fragment@sha256:test", + "display_name": "Red Hat OpenShift AI", + "publisher": "RHOAI Development Catalog", + "pull_secret": {"vault": {"name": "psap-rhoai-rc", "content": "rhoai_rc.secret"}}, + "staging_pull_secret": { + "vault": { + "name": "psap-forge-staging-image-pull", + "content": ".dockerconfigjson", + } + }, + } + ) + ) + + assert ( + merged_payload["auths"]["quay.io/rhoai"]["auth"] + == "rhoai+rhoai_external_readonly_bot:token-c" + ) + assert merged_payload["auths"]["registry.stage.redhat.io/rhaii"]["auth"] == "token-a" + assert ( + merged_payload["auths"]["registry.stage.redhat.io/rhaii-early-access"]["auth"] == "token-b" + ) def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: @@ -220,14 +546,108 @@ def test_model_and_deployment_profile_accept_yaml_list_strings() -> None: ] assert run_specs[0].model_slug == "openai-gpt-oss-120b" assert run_specs[2].model_slug == "qwen-qwen3-0-6b" + # With a single benchmark_key (or null), benchmark fields reflect the scalar + assert all(spec.benchmark_key is not None or spec.benchmark_slug is None for spec in run_specs) -def test_runtime_rejects_legacy_model_key_path() -> None: +def test_3d_matrix_with_multiple_benchmark_keys() -> None: _init_project_config() - core_config.project.config.setdefault("runtime", {})["model_key"] = "qwen3-0-6b" - with pytest.raises(ValueError, match="runtime.model_key"): - runtime_config.get_run_specs() + core_config.project.set_config( + "runtime.model_name", + ["openai/gpt-oss-120b", "Qwen/Qwen3-0.6B"], + ) + core_config.project.set_config( + "runtime.deployment_profile", + ["distributed-default", "precise-prefix-cache"], + ) + core_config.project.set_config( + "runtime.benchmark_key", + ["concurrent-1k-1k", "multi-turn"], + ) + + run_specs = runtime_config.get_run_specs() + + # 2 models x 2 profiles x 2 benchmarks = 8 specs + assert len(run_specs) == 8 + assert all(spec.benchmark_key is not None for spec in run_specs) + assert all(spec.benchmark_slug is not None for spec in run_specs) + + assert run_specs[0].model_name == "openai/gpt-oss-120b" + assert run_specs[0].deployment_profile_name == "distributed-default" + assert run_specs[0].benchmark_key == "concurrent-1k-1k" + + +def test_null_benchmark_key_produces_smoke_only_specs() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key is None + assert run_specs[0].benchmark_slug is None + + +def test_single_benchmark_key_backward_compatible() -> None: + _init_project_config() + + core_config.project.apply_preset("smoke") + + run_specs = runtime_config.get_run_specs() + + assert len(run_specs) == 1 + assert run_specs[0].benchmark_key == "short" + assert run_specs[0].benchmark_slug == "short" + assert run_specs[0].artifact_dirname == "llmd__short__approximate-prefix-cache" + + +def test_activate_run_spec_sets_benchmark_key() -> None: + _init_project_config() + + core_config.project.set_config( + "runtime.benchmark_key", + "[concurrent-1k-1k, multi-turn]", + ) + + run_specs = runtime_config.get_run_specs() + + for run_spec in run_specs: + with runtime_config.activate_run_spec(run_spec): + keys = runtime_config.get_benchmark_keys() + assert keys == [run_spec.benchmark_key] + assert runtime_config.get_benchmark_config() is not None + + +def test_benchmark_deployment_overrides() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + core_config.project.config["workloads"]["benchmarks"]["concurrent-1k-1k"][ + "deployment_overrides" + ] = {"vllm_args": ["--max-model-len=8192"]} + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {"vllm_args": ["--max-model-len=8192"]} + + +def test_benchmark_deployment_overrides_empty_when_not_set() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", "concurrent-1k-1k") + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} + + +def test_benchmark_deployment_overrides_empty_when_no_benchmark() -> None: + _init_project_config() + + core_config.project.set_config("runtime.benchmark_key", None) + + overrides = runtime_config.get_benchmark_deployment_overrides() + assert overrides == {} def test_render_uses_sanitized_model_name_and_profile_resources() -> None: @@ -253,48 +673,9 @@ def test_render_uses_sanitized_model_name_and_profile_resources() -> None: "requests": {"nvidia.com/gpu": "1"}, "limits": {"nvidia.com/gpu": "1"}, } - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--max-model-len=4096", - "--gpu-memory-utilization=0.92", - "--trust-remote-code", - "--no-enable-log-requests", - "--enable-prefix-caching", - "--tensor-parallel-size=1", - ] assert manifest["spec"]["router"]["scheduler"] == {} -def test_render_preserves_explicit_tensor_parallel_arg() -> None: - _init_project_config() - core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["explicit-tp"] = { - "replicas": 1, - "tensor_parallelism": 2, - "scheduler": {}, - "vllm_args": [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ], - } - core_config.project.set_config("runtime.model_name", "openai/gpt-oss-120b") - core_config.project.set_config("runtime.deployment_profile", "explicit-tp") - - manifest = render_inference_service_from_parts( - config_dir=str(PROJECT_ORCHESTRATION_DIR), - namespace="forge-llm-d", - inference_service=runtime_config.get_platform_config()["inference_service"], - model_name=runtime_config.get_model_name(), - model_slug=runtime_config.get_model_slug(), - deployment_profile=runtime_config.get_deployment_profile(), - model_cache=runtime_config.get_model_cache_config(), - ) - - assert manifest["spec"]["template"]["containers"][0]["args"] == [ - "--tensor-parallel-size=4", - "--gpu-memory-utilization=0.85", - ] - - def test_render_uses_embedded_scheduler_config() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) @@ -319,7 +700,7 @@ def test_render_uses_embedded_scheduler_config() -> None: def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> None: _init_project_config() core_config.project.set_config("model_cache.enabled", False) - core_config.project.config["deployments"]["no-scheduler"] = { + core_config.project.config["deployments"]["profiles"]["no-scheduler"] = { "replicas": 1, "tensor_parallelism": 1, "scheduler": None, @@ -341,12 +722,14 @@ def test_render_removes_scheduler_when_deployment_requests_null_scheduler() -> N assert "scheduler" not in manifest["spec"]["router"] -def test_benchmark_job_names_collapse_shared_default() -> None: +def test_benchmark_job_name_from_activated_spec() -> None: _init_project_config() core_config.project.apply_preset("gpt-oss-120b-inference-scheduling-release") - # Three benchmark_keys, all sharing the workload default job_name -> one entry. - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] + + for run_spec in runtime_config.get_run_specs(): + with runtime_config.activate_run_spec(run_spec): + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" def test_smoke_preset_benchmark_behavior() -> None: @@ -355,43 +738,10 @@ def test_smoke_preset_benchmark_behavior() -> None: core_config.project.apply_preset("smoke") # The smoke preset enables the short benchmark assert runtime_config.get_benchmark_keys() == ["short"] - assert runtime_config.get_benchmark_job_names() == ["guidellm-benchmark"] - - -def test_run_spec_preserves_namespace_managed_flag() -> None: - _init_project_config() - - core_config.project.apply_preset("smoke") - base_managed = runtime_config.get_namespace_is_managed() run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed == base_managed - - with runtime_config.activate_run_spec(run_spec): - # Inside the active spec the override is applied, but managed-ness must - # still reflect the base value, not flip to False. - assert runtime_config.get_namespace_is_managed() == base_managed - # And it is restored on exit. - assert runtime_config.get_namespace_is_managed() == base_managed - - -def test_run_spec_preserves_managed_flag_when_namespace_is_auto_derived() -> None: - """Verify managed flag stays True even when activate_run_spec sets namespace_override.""" - _init_project_config() - - # Force auto-derived namespace by clearing the hardcoded name - core_config.project.config["platform"]["cluster"]["namespace"]["name"] = None - - core_config.project.apply_preset("smoke") - assert runtime_config.get_namespace_is_managed() is True - - run_spec = runtime_config.get_run_specs()[0] - assert run_spec.namespace_is_managed is True - with runtime_config.activate_run_spec(run_spec): - # activate_run_spec sets namespace_override, but managed flag must stay True - assert runtime_config.get_namespace_is_managed() is True - assert runtime_config.get_namespace_is_managed() is True + assert runtime_config.get_benchmark_job_name() == "guidellm-benchmark" @pytest.mark.parametrize( diff --git a/projects/llm_d/toolbox/capture_prepare_state/main.py b/projects/llm_d/toolbox/capture_prepare_state/main.py index 8d38c8783..f1e7a1248 100644 --- a/projects/llm_d/toolbox/capture_prepare_state/main.py +++ b/projects/llm_d/toolbox/capture_prepare_state/main.py @@ -6,7 +6,6 @@ from pathlib import Path from projects.core.dsl import entrypoint, execute_tasks, task -from projects.core.dsl.utils import write_text from projects.core.dsl.utils.k8s import oc logger = logging.getLogger(__name__) @@ -65,10 +64,10 @@ def capture_datasciencecluster_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured DataScienceCluster: {destination}" else: logger.warning("Failed to capture DataScienceCluster: %s", result.stderr) @@ -91,10 +90,10 @@ def capture_gateway_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway: {destination}" else: logger.warning("Failed to capture Gateway: %s", result.stderr) @@ -117,10 +116,10 @@ def capture_gateway_service_state(args, ctx): "yaml", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured Gateway service: {destination}" else: logger.warning("Failed to capture Gateway service: %s", result.stderr) @@ -144,10 +143,10 @@ def capture_namespace_events(args, ctx): "--sort-by=.metadata.creationTimestamp", check=False, log_stdout=False, + stdout_dest=destination, ) - if result.returncode == 0 and result.stdout: - write_text(destination, result.stdout) + if result.returncode == 0: return f"Captured namespace events: {destination}" else: logger.warning("Failed to capture namespace events: %s", result.stderr) diff --git a/projects/llm_d/toolbox/cleanup_test_resources/main.py b/projects/llm_d/toolbox/cleanup_test_resources/main.py index 90e12d77a..464c9f420 100644 --- a/projects/llm_d/toolbox/cleanup_test_resources/main.py +++ b/projects/llm_d/toolbox/cleanup_test_resources/main.py @@ -6,7 +6,7 @@ import subprocess from projects.core.dsl import entrypoint, execute_tasks, retry, shell, task -from projects.core.dsl.utils.k8s import oc, oc_get_json, oc_resource_exists +from projects.core.dsl.utils.k8s import oc, oc_get_json logger = logging.getLogger("DSL") @@ -164,33 +164,42 @@ def delete_llm_d_labeled_resources(args, ctx): def delete_inference_service(args, ctx): """Delete the LLM inference service""" - _best_effort_delete( - "llminferenceservice", + result = oc( "delete", "llminferenceservice", args.inference_service_name, "-n", args.namespace, "--ignore-not-found=true", + "--timeout=30s", + check=False, ) + if result.returncode != 0: + logger.error( + "Failed to delete llminferenceservice/%s (rc=%d), capturing describe", + args.inference_service_name, + result.returncode, + ) + oc( + "describe", + "llminferenceservice", + args.inference_service_name, + "-n", + args.namespace, + check=False, + ) + raise RuntimeError( + f"Failed to delete llminferenceservice/{args.inference_service_name} " + f"in {args.namespace} (rc={result.returncode})" + ) - return f"Deleted llminferenceservice {args.inference_service_name}" - - -@retry(attempts=90, delay=10, backoff=1.0) -@task -def wait_for_inference_service_deletion(args, ctx): - """Wait for the llminferenceservice to be deleted""" - - if not oc_resource_exists( - "llminferenceservice", args.inference_service_name, namespace=args.namespace - ): - return f"llminferenceservice/{args.inference_service_name} deleted from {args.namespace}" - - return ( - False, - f"Waiting for llminferenceservice/{args.inference_service_name} deletion in {args.namespace}", - ) + # Check if something was actually deleted by looking at the output + if result.stdout.strip(): + return f"Deleted llminferenceservice {args.inference_service_name}" + else: + return ( + f"LLMInferenceService {args.inference_service_name} was not found (already cleaned up)" + ) @retry(attempts=90, delay=10, backoff=1.0) @@ -218,9 +227,15 @@ def wait_for_workload_pods_deletion(args, ctx): def _best_effort_delete(description: str, *oc_args: str) -> None: - """Best effort deletion of Kubernetes resources with timeout""" + """Best effort deletion of Kubernetes resources with timeout.""" try: - oc(*oc_args, check=False) + result = oc(*oc_args, "--timeout=300s", check=False) + if result.returncode != 0: + logger.warning( + "Failed to delete %s (rc=%d), capturing describe", description, result.returncode + ) + describe_args = tuple(a for a in oc_args if a != "delete") + oc("describe", *describe_args, check=False) except subprocess.TimeoutExpired: logger.warning("Timed out deleting %s: oc %s", description, " ".join(oc_args)) diff --git a/projects/llm_d_legacy/testing/config.yaml b/projects/llm_d_legacy/testing/config.yaml index b16b0008f..dd4050ccf 100644 --- a/projects/llm_d_legacy/testing/config.yaml +++ b/projects/llm_d_legacy/testing/config.yaml @@ -364,13 +364,11 @@ caliper: kpi: enabled: false - generate: + artifacts_to_kpis: enabled: false output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json diff --git a/projects/matrix_benchmarking/subproject/PLUGINS.md b/projects/matrix_benchmarking/subproject/PLUGINS.md index 5499f7985..e98a1dc68 100644 --- a/projects/matrix_benchmarking/subproject/PLUGINS.md +++ b/projects/matrix_benchmarking/subproject/PLUGINS.md @@ -155,4 +155,4 @@ When creating a new visualization plugin: 5. Create plotting and analysis components 6. Test with sample data -For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. \ No newline at end of file +For more information on developing visualization plugins, refer to the matrix benchmarking framework documentation. diff --git a/projects/mcp_gateway/nightly_resolvers/__init__.py b/projects/mcp_gateway/nightly_resolvers/__init__.py new file mode 100644 index 000000000..73ebd8af5 --- /dev/null +++ b/projects/mcp_gateway/nightly_resolvers/__init__.py @@ -0,0 +1,5 @@ +"""MCP Gateway nightly resolvers. + +Contains ImageReceiver implementations for different image sources. +Receiver selection is config-driven via nightly.sources..receiver. +""" diff --git a/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml new file mode 100644 index 000000000..1eb501472 --- /dev/null +++ b/projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml @@ -0,0 +1,50 @@ +# CronJob to trigger the nightly pipeline for mcp_gateway (GHCR source). +# Runs daily at 2:00 AM UTC. Creates a clusterless FournosJob that +# uses the "nightly" pipeline to check for new versions on ghcr.io. +# +# Apply manually: +# oc apply -f projects/mcp_gateway/nightly_resolvers/cronjob-ghcr.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nightly-mcp-gateway-ghcr + namespace: psap-automation + labels: + fournos.dev/nightly: "true" + fournos.dev/project: mcp-gateway + fournos.dev/source: ghcr +spec: + schedule: "0 2 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: fournos + restartPolicy: Never + containers: + - name: submit + image: quay.io/openshift/origin-cli:latest + command: ["sh", "-c"] + args: + - | + cat <<'EOF' | oc apply -f - + apiVersion: fournos.dev/v1 + kind: FournosJob + metadata: + generateName: nightly-mcp-gateway- + namespace: fournos-workloads + spec: + clusterless: true + exclusive: false + pipeline: "nightly" + configOverrides: + nightly.source: "ghcr" + executionEngine: + forge: + project: "mcp_gateway" + args: [] + EOF + echo "Nightly FournosJob created for mcp_gateway (ghcr)" diff --git a/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py new file mode 100644 index 000000000..36853fc71 --- /dev/null +++ b/projects/mcp_gateway/nightly_resolvers/ghcr_receiver.py @@ -0,0 +1,106 @@ +"""GHCR image receiver for mcp_gateway. + +Resolves the latest commit SHA that has a published image on ghcr.io. +Checks recent commits from the configured repo and finds the first +one with a corresponding container image tag (sha-). + +No authentication needed for public packages (anonymous token). + +Configuration (in config.yaml under nightly.sources.ghcr): + repo: GitHub repo in "owner/name" format (e.g. "Kuadrant/mcp-gateway") + image: GHCR image path without registry (e.g. "kuadrant/mcp-gateway") +""" + +from __future__ import annotations + +import json +import logging +import time +import urllib.error +import urllib.request + +from projects.core.nightly.base_receiver import ImageReceiver + +logger = logging.getLogger(__name__) + +MAX_ATTEMPTS = 3 +RETRY_DELAY_SECONDS = 10 + + +class GHCRReceiver(ImageReceiver): + """Find the latest commit with a published ghcr.io image.""" + + NAME = "ghcr" + + def __init__(self): + from projects.core.library import config + + source_cfg = config.project.get_config("nightly.sources.ghcr") + self.repo = source_cfg["repo"] + self.image = source_cfg["image"] + + self._commits_url = f"https://api.github.com/repos/{self.repo}/commits?per_page=20" + self._token_url = ( + f"https://ghcr.io/token?service=ghcr.io&scope=repository:{self.image}:pull" + ) + self._manifest_url = f"https://ghcr.io/v2/{self.image}/manifests/" + + def get_latest_version(self) -> str: + last_error = None + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return self._fetch_version() + except urllib.error.URLError as e: + last_error = e + if attempt < MAX_ATTEMPTS: + logger.warning( + "GHCR attempt %d/%d failed: %s. Retrying in %ds...", + attempt, + MAX_ATTEMPTS, + e, + RETRY_DELAY_SECONDS, + ) + time.sleep(RETRY_DELAY_SECONDS) + raise RuntimeError(f"GHCR receiver failed after {MAX_ATTEMPTS} attempts") from last_error + + def _fetch_version(self) -> str: + token = self._get_ghcr_token() + shas = self._get_recent_commits() + + for sha in shas: + if self._image_exists(sha, token): + return sha + + raise RuntimeError( + f"No commit found with a published image on ghcr.io " + f"(checked {len(shas)} recent commits)" + ) + + def _get_ghcr_token(self) -> str: + with urllib.request.urlopen(self._token_url, timeout=30) as resp: + data = json.loads(resp.read()) + return data["token"] + + def _get_recent_commits(self) -> list[str]: + req = urllib.request.Request(self._commits_url) + with urllib.request.urlopen(req, timeout=30) as resp: + commits = json.loads(resp.read()) + return [c["sha"] for c in commits] + + def _image_exists(self, sha: str, token: str) -> bool: + tag = f"sha-{sha}" + req = urllib.request.Request( + self._manifest_url + tag, + headers={ + "Accept": "application/vnd.oci.image.index.v1+json, " + "application/vnd.docker.distribution.manifest.v2+json", + "Authorization": f"Bearer {token}", + }, + ) + try: + with urllib.request.urlopen(req, timeout=30): + return True + except urllib.error.HTTPError as e: + if e.code == 404: + return False + raise diff --git a/projects/mcp_gateway/orchestration/config.yaml b/projects/mcp_gateway/orchestration/config.yaml index 34f67a0a9..2c8b22a34 100644 --- a/projects/mcp_gateway/orchestration/config.yaml +++ b/projects/mcp_gateway/orchestration/config.yaml @@ -1,3 +1,26 @@ +nightly: + caliper_export: + experiment: nightly-pipeline-experiment + run_naming: + prefix: "mcp-gw-nightly" + single_run: "{prefix}-{timestamp}-{outcome}" + sources: + ghcr: + receiver: "projects.mcp_gateway.nightly_resolvers.ghcr_receiver.GHCRReceiver" + repo: "Kuadrant/mcp-gateway" + image: "kuadrant/mcp-gateway" + mlflow_experiment: "mcp-gw-cpt-ghcr" + verifier: "projects.caliper.orchestration.mlflow_verifier.CaliperMLflowVerifier" + mlflow_verifier: + experiment: "mcp-gw-cpt-ghcr" + version_pattern: "-v(.+?)-\\d{8}-\\d{6}$" + strip_prefix: "sha-" + cluster: "forge-smoke-testing" + version_key: "infrastructure.mcp_gateway_version" + pipeline: "forge-full" + preset: "demo" + owner: "ashtakb" + vaults: all: [] all-optional: @@ -34,14 +57,12 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false analyze: enabled: false - baseline: null + historical_kpis: null output: kpi_analyze.json export: diff --git a/projects/mcp_gateway/orchestration/notifications.py b/projects/mcp_gateway/orchestration/notifications.py index 297eb9aa8..0c4683960 100644 --- a/projects/mcp_gateway/orchestration/notifications.py +++ b/projects/mcp_gateway/orchestration/notifications.py @@ -37,13 +37,9 @@ ("mcp_gw_failure_rate", "Failure rate", "%"), ] -# Mapping from kpis.jsonl IDs to MLflow metrics.json keys (logged via mlflow.log_metric) -_KPI_TO_MLFLOW_METRIC = { - "mcp_gw_requests_per_second": "requests_per_second", - "mcp_gw_p95_ms": "p95_ms", - "mcp_gw_p99_ms": "p99_ms", - "mcp_gw_failure_rate": "failure_rate", -} +# KPI IDs are now logged directly as MLflow metric keys via the generic +# caliper kpis-to-metrics conversion — no mapping needed. +_KPI_IDS = {kpi_id for kpi_id, _, _ in TARGET_KPIS} _RUN_NAME_PREFIX = "forge-mcp-gateway-" @@ -376,12 +372,7 @@ def _load_previous_kpis_from_mlflow( return {}, "", True raw_metrics = previous_run.data.metrics or {} - mlflow_to_kpi = {v: k for k, v in _KPI_TO_MLFLOW_METRIC.items()} - metrics = {} - for mlflow_key, value in raw_metrics.items(): - kpi_id = mlflow_to_kpi.get(mlflow_key) - if kpi_id: - metrics[kpi_id] = value + metrics = {k: v for k, v in raw_metrics.items() if k in _KPI_IDS} return metrics, prev_name, False diff --git a/projects/mcp_gateway/orchestration/pr_args.py b/projects/mcp_gateway/orchestration/pr_args.py index 82cffdb40..1523160bd 100644 --- a/projects/mcp_gateway/orchestration/pr_args.py +++ b/projects/mcp_gateway/orchestration/pr_args.py @@ -9,6 +9,8 @@ /test fournos mcp_gateway smoke /test fournos mcp_gateway smoke collect_cluster_info /version 0.7.0 + /pipeline nightly ghcr + /pipeline nightly quay """ from __future__ import annotations @@ -77,4 +79,24 @@ def parse_project_directives(comment_text: str) -> tuple[dict[str, Any], list[st parsed_directives.append(line) logger.info("Parsed mcp_gateway version: %s", parts[0]) + # Parse /pipeline nightly — extract source from pipeline directive + # The directive line itself is already captured by the FOURNOS /pipeline handler, + # so only extract the config overrides here to avoid duplicating it in notifications. + if line.startswith("/pipeline nightly "): + parts = line.split() + if len(parts) == 3: + source = parts[2] + config_overrides["nightly.source"] = source + config_overrides["fournos.job.pipeline_name"] = "nightly" + config_overrides["caliper.export.backend.mlflow.config.experiment"] = ( + "nightly-pipeline-experiment" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.prefix"] = ( + "mcp-gw-nightly" + ) + config_overrides["caliper.export.backend.mlflow.config.run_naming.single_run"] = ( + "{prefix}-{timestamp}-{outcome}" + ) + logger.info("Parsed nightly source from pipeline directive: %s", source) + return config_overrides, parsed_directives diff --git a/projects/mcp_gateway/orchestration/presets.d/presets.yaml b/projects/mcp_gateway/orchestration/presets.d/presets.yaml index 55fe683ba..19c1267c3 100644 --- a/projects/mcp_gateway/orchestration/presets.d/presets.yaml +++ b/projects/mcp_gateway/orchestration/presets.d/presets.yaml @@ -61,4 +61,4 @@ large-scale: experiment.target: [gateway] experiment.duration_seconds: 240 experiment.warmup_seconds: 60 - experiment.tools_per_server: 10 \ No newline at end of file + experiment.tools_per_server: 10 diff --git a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py index 107f60ae5..6fd9e2cdf 100644 --- a/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py +++ b/projects/mcp_gateway/postprocess/mcp_gateway/parsing/parsers.py @@ -2,9 +2,7 @@ from __future__ import annotations -import json import logging -from pathlib import Path from typing import Any from projects.agentic_tools.locust.helpers.parse_results import RunMetrics, parse_stats_csv @@ -17,8 +15,6 @@ logger = logging.getLogger(__name__) STATS_CSV = "stats.csv" -METRICS_FILE = "metrics.json" -PARAMETERS_FILE = "parameters.json" def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: @@ -33,7 +29,7 @@ def _labels_from_node(node: TestBaseNode) -> dict[str, Any]: def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: - """Convert RunMetrics to a flat dictionary suitable for metrics.json.""" + """Convert RunMetrics to a flat dictionary for unified result records.""" return { "total_requests": metrics.total_requests, "total_failures": metrics.total_failures, @@ -48,13 +44,6 @@ def _run_metrics_to_dict(metrics: RunMetrics) -> dict[str, Any]: } -def _write_json(path: Path, data: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8") as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write("\n") - - class MCPGatewayParser: """Parser for Locust stats.csv artifacts from MCP Gateway tests.""" @@ -89,11 +78,6 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: labels = _labels_from_node(node) metrics_dict = _run_metrics_to_dict(run_metrics) - _write_json(node.directory / METRICS_FILE, metrics_dict) - - params = {str(k): ("" if v is None else str(v)) for k, v in labels.items()} - _write_json(node.directory / PARAMETERS_FILE, params) - records.append( UnifiedResultRecord( test_base_path=str(node.test_path), diff --git a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py index 78f45ab95..d45287aab 100644 --- a/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py +++ b/projects/mcp_gateway/postprocess/tests/test_mcp_gateway_plugin.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path import pytest @@ -66,30 +65,14 @@ def test_parse_creates_records(self, tmp_path: Path): assert record.metrics["requests_per_second"] == 31.5 assert record.metrics["p95_ms"] == 80.0 - def test_parse_writes_metrics_json(self, tmp_path: Path): + def test_parse_does_not_write_metrics_json(self, tmp_path: Path): + """Parser no longer writes metrics.json — caliper handles it generically.""" node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) parser = MCPGatewayParser() parser.parse(tmp_path, [node]) - metrics_file = tmp_path / "run-a" / "metrics.json" - assert metrics_file.exists() - data = json.loads(metrics_file.read_text()) - assert data["total_requests"] == 1000 - assert data["requests_per_second"] == 31.5 - - def test_parse_writes_parameters_json(self, tmp_path: Path): - node = _make_test_node(tmp_path, "run-a", SAMPLE_STATS_CSV, TEST_LABELS) - parser = MCPGatewayParser() - - parser.parse(tmp_path, [node]) - - params_file = tmp_path / "run-a" / "parameters.json" - assert params_file.exists() - data = json.loads(params_file.read_text()) - assert data["preset"] == "smoke" - assert data["target"] == "gateway" - assert data["users"] == "16" + assert not (tmp_path / "run-a" / "metrics.json").exists() def test_parse_no_stats_csv(self, tmp_path: Path): node_dir = tmp_path / "run-empty" @@ -123,8 +106,8 @@ def test_parse_multiple_nodes(self, tmp_path: Path): assert "run-b" in paths for node_name in ["run-a", "run-b"]: - assert (tmp_path / node_name / "metrics.json").exists() - assert (tmp_path / node_name / "parameters.json").exists() + assert not (tmp_path / node_name / "metrics.json").exists() + assert not (tmp_path / node_name / "parameters.json").exists() # --------------------------------------------------------------------------- diff --git a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py index 353298467..062efe707 100644 --- a/projects/mcp_gateway/toolbox/apply_infrastructure/main.py +++ b/projects/mcp_gateway/toolbox/apply_infrastructure/main.py @@ -253,6 +253,4 @@ def _generate_infrastructure_manifest( def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) diff --git a/projects/mcp_gateway/toolbox/cleanup_platform/main.py b/projects/mcp_gateway/toolbox/cleanup_platform/main.py index bbfe8ced2..f28860a0a 100644 --- a/projects/mcp_gateway/toolbox/cleanup_platform/main.py +++ b/projects/mcp_gateway/toolbox/cleanup_platform/main.py @@ -200,13 +200,26 @@ def capture_cleanup_state(args, ctx): artifacts_dir = args.artifact_dir / "artifacts" artifacts_dir.mkdir(parents=True, exist_ok=True) - result = oc("get", "csv", "-A", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-csvs.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "csv", + "-A", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-csvs.txt", + ) - result = oc("get", "namespaces", "-o", "wide", check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - (artifacts_dir / "remaining-namespaces.txt").write_text(result.stdout, encoding="utf-8") + oc( + "get", + "namespaces", + "-o", + "wide", + check=False, + log_stdout=False, + stdout_dest=artifacts_dir / "remaining-namespaces.txt", + ) return "Captured post-cleanup state" diff --git a/projects/mcp_gateway/toolbox/install_platform/main.py b/projects/mcp_gateway/toolbox/install_platform/main.py index fcf5c5d5c..40179036a 100644 --- a/projects/mcp_gateway/toolbox/install_platform/main.py +++ b/projects/mcp_gateway/toolbox/install_platform/main.py @@ -115,8 +115,8 @@ def install_service_mesh_instance(args, ctx): if "wait_for_ready" in step: spec = step["wait_for_ready"] - ctx._mesh_ready_spec = spec - return (False, "Service Mesh instance applied, readiness checked in next task") + ctx.mesh_ready_spec = spec + return "Service Mesh instance applied, readiness checked in next task" return f"Service Mesh instance applied from {kustomize_path}" @@ -125,7 +125,7 @@ def install_service_mesh_instance(args, ctx): @task def wait_service_mesh_ready(args, ctx): """Wait for Istio instance to report Ready condition""" - spec = getattr(ctx, "_mesh_ready_spec", None) + spec = getattr(ctx, "mesh_ready_spec", None) if not spec: return "No readiness spec, skipping" @@ -524,9 +524,7 @@ def _get_mcp_host() -> str: def _capture_to_file(path: Path, *oc_args: str) -> None: """Best-effort capture of oc output to a file.""" - result = oc(*oc_args, check=False, log_stdout=False) - if result.returncode == 0 and result.stdout: - path.write_text(result.stdout, encoding="utf-8") + oc(*oc_args, check=False, log_stdout=False, stdout_dest=path) def _parse_version(version: str) -> tuple[int, ...]: diff --git a/projects/rhaiis/README.md b/projects/rhaiis/README.md index f0e425e8f..5991f39ff 100644 --- a/projects/rhaiis/README.md +++ b/projects/rhaiis/README.md @@ -3,31 +3,48 @@ `rhaiis` is the Forge project for benchmarking AI inference engines on OpenShift using KServe InferenceService. -The current implementation focuses on single-replica vLLM deployments benchmarked -with GuideLLM. The workflow deploys an InferenceService, waits for readiness, -runs a GuideLLM benchmark job, extracts results, and cleans up. +Supports **vLLM**, **SGLang**, and **TRT-LLM** engines via a generic abstraction. +The workflow deploys an InferenceService with the selected engine, runs warmup +and/or profiler passes, executes multi-profile GuideLLM benchmarks, generates a +dashboard CSV via the Caliper postprocessing pipeline, syncs results to S3, and +cleans up. ## Workflow sequence ``` -deploy_kserve_isvc Deploy ServingRuntime + InferenceService +deploy_kserve_isvc Deploy ServingRuntime + InferenceService | -wait_isvc_ready Poll InferenceService status + health check +wait_isvc_ready Poll InferenceService status + health check | -run_guidellm_benchmark Create PVC, run GuideLLM job, copy results, cleanup + ┌─────┴─────┐ + │ Phase 1 │ For each workload profile: + │ (warmup │ - warmup run (optional) OR profiler-gated run + │ /profiler)│ - profiler trace upload to S3 + └─────┬─────┘ + ┌─────┴─────┐ + │ Phase 2 │ For each workload profile (in its own artifact dir): + │ (benchmark)│ - write per-workload test labels + │ │ - run GuideLLM benchmark at configured rates + └─────┬─────┘ | -capture_isvc_state Capture ISVC YAML, pod logs, events +capture_isvc_state Capture ISVC YAML, pod logs, events | -cleanup_isvc Delete InferenceService + ServingRuntime +cleanup_isvc Delete InferenceService + ServingRuntime + | +Caliper postprocessing Parse → KPI → CSV export (per-profile rows) + | +S3 dashboard sync Append CSV to consolidated dashboard on S3 + | +Regression analysis Compare against baseline version (optional) ``` On failure at any step, `capture_isvc_state` and `cleanup_isvc` still run -(try/finally in the orchestration layer). +(try/finally in the orchestration layer). Pipeline failures and non-critical +warnings are reported via Slack. -After the benchmark, `run_and_postprocess()` automatically runs: -- KPI computation (22 metrics via rhaiis postprocess plugin) -- PSAP payload generation (model-furnace compatible JSON) -- MLflow metadata tagging +Each workload profile runs in its own `NextArtifactDir` with per-workload test +labels, so the Caliper parser creates separate records per profile. This ensures +each profile gets its own dashboard CSV rows. The benchmark step uses the canonical `projects.guidellm.toolbox.run_guidellm_benchmark` (shared with llm_d). rhaiis builds `guidellm_args` from its config and passes them @@ -37,8 +54,8 @@ to the canonical runner. Config is split into `config.yaml` (base) and `config.d/` (per-domain): -- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper -- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, vLLM args, images, gpu_types +- [`orchestration/config.yaml`](./orchestration/config.yaml) — vaults, benchmarks, tests, caliper postprocessing +- [`orchestration/config.d/rhaiis.yaml`](./orchestration/config.d/rhaiis.yaml) — deploy defaults, engine config (vLLM/SGLang/TRT-LLM), images, gpu_types, S3 settings, profiler - [`orchestration/config.d/models.yaml`](./orchestration/config.d/models.yaml) — 52 model definitions - [`orchestration/config.d/workloads.yaml`](./orchestration/config.d/workloads.yaml) — workload profiles @@ -46,11 +63,43 @@ Key sections: | Section | Purpose | |---------|---------| -| `rhaiis` | Namespace, accelerator, vLLM images, deploy settings, vLLM args | -| `models` | Model definitions (hf_model_id, per-model vLLM overrides) | +| `rhaiis` | Namespace, accelerator, deploy settings, S3 config | +| `rhaiis.engine` | Active engine: `vllm` (default), `sglang`, or `trtllm` | +| `rhaiis.engines.{vllm,sglang,trtllm}` | Per-engine images, port, and default args | +| `rhaiis.engines.trtllm.trtllm_config` | TRT-LLM server config (KV cache, CUDA graphs, MoE) | +| `rhaiis.deploy` | Deploy settings (replicas, CPU/memory, image_pull_secrets list, storage) | +| `rhaiis.s3` | S3 bucket, vault, and credentials for dashboard CSV and profiler trace uploads | +| `rhaiis.profiler` | PyTorch profiler settings (enable, S3 prefix, rates, labels) | +| `models` | Model definitions (hf_model_id, per-model `engine_args` overrides) | | `workloads` | Benchmark profiles (data shape, rates, max_seconds) | -| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size | -| `tests` | CI test mapping (model_key, workload_key) | +| `benchmarks.guidellm` | GuideLLM image, backend, timeout, PVC size, HF token secret, fs_group | +| `tests` | CI test mapping (model_key, workload_keys, version) | +| `caliper.postprocess` | Caliper postprocessing pipeline (parse, KPI, CSV export) | + +## Engine support + +rhaiis supports three inference engines via a generic abstraction: + +| Engine | Config key | Image source | TP arg | Notes | +|--------|-----------|-------------|--------|-------| +| vLLM | `rhaiis.engine: vllm` (default) | `rhaiis.engines.vllm.images` | `tensor-parallel-size` | No root required | +| SGLang | `rhaiis.engine: sglang` | `rhaiis.engines.sglang.images` | `tp-size` | No root required, uses `sglang serve` entrypoint | +| TRT-LLM | `rhaiis.engine: trtllm` | `rhaiis.engines.trtllm.images` | `tp_size` | Requires root (`anyuid` SCC), NGC pull secret | + +Models define `engine_args` with vLLM-style keys (e.g. `tensor-parallel-size`). +When running with SGLang or TRT-LLM, arguments are automatically translated +(e.g. `tensor-parallel-size` → `tp-size` for SGLang, `tp_size` for TRT-LLM). + +Engine-specific args can also be set directly via `rhaiis.engines..args.*` +in FournosJob `configOverrides`, which take precedence over model-level `engine_args`. + +TRT-LLM additionally supports a `trtllm_config` block for server-side configuration +(KV cache, CUDA graphs, MoE backend, etc.) that is serialized as a JSON config file +inside the container. + +KServe manifests (ServingRuntime + InferenceService) are built as Python dicts in +`orchestration/manifests.py` with engine-specific container construction, then +passed to the generic `deploy_kserve_isvc` tool for application. ## Fournos integration @@ -58,38 +107,156 @@ rhaiis supports Fournos-driven execution via `ci.py`: ``` bin/run_ci rhaiis ci resolve-fournos-config # Populate spec.secretRefs + hardware -bin/run_ci rhaiis ci pre-cleanup # Delete leftover jobs/pods +bin/run_ci rhaiis ci pre-cleanup # No-op (avoid cleaning running resources) bin/run_ci rhaiis ci prepare # Verify cluster, ensure namespace/SA +bin/run_ci rhaiis ci preflight # Preflight checks bin/run_ci rhaiis ci test # Deploy, benchmark, capture, cleanup +bin/run_ci rhaiis ci post-cleanup # Detect pipeline failures, cleanup resources bin/run_ci rhaiis ci export-artifacts # Caliper export to MLflow ``` -FournosJob example: +### FournosJob YAML (vLLM) + ```yaml apiVersion: fournos.dev/v1 kind: FournosJob metadata: generateName: rhaiis-benchmark- spec: - owner: mehul - displayName: rhaiis-qwen3-benchmark - pipeline: forge-prepare-test - cluster: fournos-fg - priority: manual + owner: haumesh + displayName: rhaiis-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + rhaiis.engines.vllm.images.nvidia: vllm/vllm-openai:v0.24.0 + tests.rhaiis.version: "vLLM-0.24.0" + tests.rhaiis.workload_keys: ["profile1","profile2","profile4"] + rhaiis.engines.vllm.args.tensor-parallel-size: 2 + rhaiis.cluster_tag: "zeus2" + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 # opt-in for IBM Cloud clusters + env: + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 +``` + +### FournosJob YAML (SGLang) + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: sglang-benchmark- +spec: + owner: haumesh + displayName: sglang-benchmark + pipeline: forge-full + cluster: zeus hardware: - gpuType: l40s - gpuCount: 1 + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications executionEngine: forge: project: rhaiis - args: [nvidia, profile1] + args: [nvidia] configOverrides: - tests.rhaiis.model_key: qwen3-0_6b - rhaiis.namespace: kserve-e2e-perf - rhaiis.deploy.image_pull_secret: npalaska-image-pull - rhaiis.deploy.storage_pvc: model-pvc + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "SGLang-0.5.11" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: sglang + rhaiis.engines.sglang.args.tp-size: 2 + rhaiis.engines.sglang.args.disable-radix-cache: true + rhaiis.engines.sglang.args.mem-fraction-static: 0.90 + rhaiis.engines.sglang.args.context-length: 8192 + rhaiis.engines.sglang.args.trust-remote-code: true + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull"] + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 + env: + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 ``` +### FournosJob YAML (TRT-LLM) + +TRT-LLM requires the `anyuid` SCC on the target cluster (`oc adm policy add-scc-to-user anyuid -z default -n `). + +```yaml +apiVersion: fournos.dev/v1 +kind: FournosJob +metadata: + generateName: trtllm-benchmark- +spec: + owner: haumesh + displayName: trtllm-benchmark + pipeline: forge-full + cluster: zeus + hardware: + gpuType: h200 + gpuCount: 2 + secretRefs: + - psap-forge-dashboard-s3 + - psap-forge-notifications + executionEngine: + forge: + project: rhaiis + args: [nvidia] + configOverrides: + tests.rhaiis.model_key: nemotron3super-120b-fp8 + tests.rhaiis.version: "TRT-LLM-1.3.0rc13" + tests.rhaiis.workload_keys: ["profile1"] + rhaiis.engine: trtllm + rhaiis.deploy.image_pull_secrets: ["npalaska-image-pull", "ngc-secret"] + rhaiis.deploy.memory_request: "256Gi" + # Engine args + rhaiis.engines.trtllm.args.tp_size: 2 + rhaiis.engines.trtllm.args.ep_size: 2 + rhaiis.engines.trtllm.args.max_batch_size: 256 + rhaiis.engines.trtllm.args.max_num_tokens: 8192 + rhaiis.engines.trtllm.args.trust_remote_code: true + # TRT-LLM server config + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.dtype: fp8 + rhaiis.engines.trtllm.trtllm_config.kv_cache_config.free_gpu_memory_fraction: 0.8 + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.enable_padding: true + rhaiis.engines.trtllm.trtllm_config.cuda_graph_config.max_batch_size: 256 + rhaiis.engines.trtllm.trtllm_config.enable_attention_dp: true + rhaiis.engines.trtllm.trtllm_config.moe_config.backend: TRTLLM + caliper.postprocess.csv_dashboard.enabled: true + benchmarks.guidellm.fs_group: 0 + env: + PULL_PULL_SHA: "" # e.g. forge-rhaiis-v0.0.2 +``` + +### GitHub PR workflow + +Jobs can also be triggered via PR comments on `openshift-psap/forge`: + +``` +/test fournos rhaiis nvidia +/pipeline forge-full +/cluster zeus +/var tests.rhaiis.model_key: nemotron3super-120b-fp8 +/var tests.rhaiis.version: vLLM-0.24.0 +/var tests.rhaiis.workload_keys: ["profile1"] +/var rhaiis.engines.vllm.args.tensor-parallel-size: 2 +``` + +Note: `/var` directives use `key: value` format (colon required). +If `PULL_PULL_SHA` is not set, the PR HEAD commit is used automatically. + Presets from `args` are applied via `project.args` → `presets.d/presets.yaml`. Config overrides use dot-notation to set any nested config value. @@ -98,13 +265,28 @@ Available configOverrides: | Key | Description | |-----|-------------| | `tests.rhaiis.model_key` | Model key from config.d/models.yaml | +| `tests.rhaiis.version` | Version label for dashboard and regression | +| `tests.rhaiis.workload_keys` | List of workload profiles to run | +| `tests.rhaiis.warmup` | Enable warmup pass before benchmarks | +| `tests.rhaiis.slack_user` | Slack user ID for failure notifications | +| `tests.rhaiis.compare_version` | Baseline version for regression comparison | +| `rhaiis.engine` | Inference engine: `vllm` (default), `sglang`, `trtllm` | | `rhaiis.namespace` | Kubernetes namespace | -| `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `rhaiis.cluster_tag` | Cluster identifier for dashboard grouping | +| `rhaiis.deploy.image_pull_secrets` | List of image pull secret names | | `rhaiis.deploy.storage_pvc` | PVC name for model storage | | `rhaiis.deploy.replicas` | Number of predictor replicas | -| `rhaiis.images.nvidia` | vLLM container image override | -| `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | +| `rhaiis.deploy.memory_request` | Memory request for predictor (e.g. `256Gi` for TRT-LLM) | +| `rhaiis.engines.vllm.args.*` | vLLM CLI args (e.g. `tensor-parallel-size`, `gpu-memory-utilization`) | +| `rhaiis.engines.sglang.args.*` | SGLang CLI args (e.g. `tp-size`, `mem-fraction-static`, `context-length`) | +| `rhaiis.engines.trtllm.args.*` | TRT-LLM CLI args (e.g. `tp_size`, `ep_size`, `max_batch_size`) | +| `rhaiis.engines.trtllm.trtllm_config.*` | TRT-LLM server config (kv_cache, cuda_graph, moe) | +| `rhaiis.profiler.enabled` | Enable PyTorch profiler | +| `rhaiis.agent_analysis.enabled` | Enable AI agent regression analysis | +| `caliper.postprocess.csv_dashboard.enabled` | Enable dashboard CSV S3 sync | | `benchmarks.guidellm.timeout` | Benchmark timeout in seconds | +| `benchmarks.guidellm.hf_token_secret` | K8s secret name for HF_TOKEN (omit to skip) | +| `benchmarks.guidellm.fs_group` | Pod-level fsGroup for PVC permissions (disabled by default) | Monitoring Fournos jobs: ```bash @@ -122,16 +304,24 @@ oc patch fournosjob -n psap-automation \ - CLI: [`orchestration/cli.py`](./orchestration/cli.py) - CI: [`orchestration/ci.py`](./orchestration/ci.py) (Fournos pipeline) - CI test: [`orchestration/test_rhaiis.py`](./orchestration/test_rhaiis.py) +- Test phase: [`orchestration/test_phase.py`](./orchestration/test_phase.py) +- Manifests: [`orchestration/manifests.py`](./orchestration/manifests.py) (KServe ServingRuntime + InferenceService builders) +- Runtime config: [`orchestration/runtime_config.py`](./orchestration/runtime_config.py) (engine-aware config helpers) +- Analysis: [`orchestration/analysis.py`](./orchestration/analysis.py) +- Notifications: [`orchestration/notifications.py`](./orchestration/notifications.py) ## Toolbox commands | Command | Source | Purpose | |---------|--------|---------| -| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Render and apply KServe InferenceService + ServingRuntime | +| `deploy_kserve_isvc` | [rhaiis](./toolbox/deploy_kserve_isvc/) | Apply pre-built KServe InferenceService + ServingRuntime manifests | | `wait_isvc_ready` | [rhaiis](./toolbox/wait_isvc_ready/) | Poll InferenceService readiness with health check | | `run_guidellm_benchmark` | [canonical](../guidellm/toolbox/run_guidellm_benchmark/) | Run GuideLLM benchmark (shared with llm_d) | | `capture_isvc_state` | [rhaiis](./toolbox/capture_isvc_state/) | Capture InferenceService YAML, pod logs, events, describe output | | `cleanup_isvc` | [rhaiis](./toolbox/cleanup_isvc/) | Delete InferenceService, ServingRuntime, wait for deletion | +| `enable_profiler_gate` | [rhaiis](./toolbox/enable_profiler_gate/) | Enable/disable the PyTorch profiler gate file on the vLLM pod | +| `verify_profiler_prereqs` | [rhaiis](./toolbox/verify_profiler_prereqs/) | Verify profiler prerequisites (gate file, sitecustomize.py) | +| `copy_profiler_traces` | [rhaiis](./toolbox/copy_profiler_traces/) | Copy Chrome trace JSON files from the vLLM pod | ## Usage @@ -196,80 +386,78 @@ Available overrides: |------|---------------|-------------| | `--rates` | `workloads..rates` | Comma-separated concurrency levels (e.g. `1,5,50`) | | `--max-seconds` | `workloads..max_seconds` | Max benchmark duration per rate | -| `--tensor-parallel` | `rhaiis.vllm_args.tensor-parallel-size` | Tensor parallel size | -| `--vllm-image` | `rhaiis.images.` | vLLM container image | +| `--tensor-parallel` | engine args | Tensor parallel size (translated per engine) | +| `--vllm-image` | `rhaiis.engines..images.` | Serving container image | | `--accelerator` | `rhaiis.accelerator` | `nvidia` or `amd` | | `--replicas` | `rhaiis.deploy.replicas` | Number of predictor replicas | | `--storage-source` | `rhaiis.deploy.storage_source` | `hf` (HuggingFace download) or `pvc` | | `--storage-pvc` | `rhaiis.deploy.storage_pvc` | PVC name for model storage | -| `--image-pull-secret` | `rhaiis.deploy.image_pull_secret` | Image pull secret name | +| `--image-pull-secret` | `rhaiis.deploy.image_pull_secrets` | Image pull secret name | | `--service-account-name` | `rhaiis.deploy.service_account_name` | Service account for predictor | | `--deployment-name` | derived from model HF ID | InferenceService name | ## Result extraction -GuideLLM results are extracted using the copy-pod pattern (same as llm_d): +GuideLLM results are extracted using `oc cp` with a gzip fallback for large files: 1. GuideLLM job writes `benchmarks.json` to a PVC mounted at `/results` 2. A copy pod is created on the same node (required for ReadWriteOnce PVC) -3. Results are extracted via `oc exec cat /results/benchmarks.json` -4. Written to local `artifacts/results/benchmarks.json` +3. Results are copied via `oc cp` from the copy pod +4. If `oc cp` fails for large files (e.g. `unexpected EOF`), the file is gzipped + inside the pod, copied compressed, and decompressed locally 5. PVC, job, and copy pod are deleted -Artifacts are stored under `/tmp/forge_/002__run_guidellm_benchmark/artifacts/`. +## Postprocessing and dashboard CSV -## Available models +After benchmarks complete, `run_and_postprocess()` runs the Caliper pipeline: -52 models from model_furnace are defined in `config.yaml`. Key families: +1. **Parse**: `RhaiisParser` extends `GuideLLMParser` with extra metrics (p1/p999 + percentiles, mean latencies, token counts, throughput) from raw `benchmarks.json` +2. **KPI generate**: `RhaiisKpiHandler` emits 33 KPI types per rate point per profile +3. **CSV export**: `RhaiisPlugin.export_kpis_to_csv` maps KPIs to the dashboard CSV + schema with all metadata columns (model, version, TP, accelerator, UUID, etc.) +4. **S3 sync**: The CSV is appended to the consolidated dashboard CSV on S3 +5. **Regression check**: Optionally compares current vs. baseline version -| Family | Key examples | TP size | -|--------|-------------|---------| -| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | -| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | -| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | -| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | -| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | -| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | -| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | -| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | -| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | -| Validation | `qwen3-0_6b` | 1 | +Dashboard CSV columns include throughput (output tok/s, total tok/s), latency +percentiles (TTFT/TPOT/ITL at median, p1, p95, p99, p999, mean), request latency, +token counts, concurrency, and run metadata. -Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` +## Notifications -## Workload profiles +Pipeline events are reported to Slack via the topsail bot: -From model_furnace `guidellm_profiles.iterations`: +- **Failure alerts**: Sent when the pipeline crashes (exceptions, image pull failures, + infrastructure errors). Includes error details, model, accelerator, TP, version, + and job ID. +- **Warning alerts**: Sent when the pipeline completes with non-critical warnings + (e.g. profiler trace upload failed, predictor log upload failed). +- **Regression alerts**: Sent when regression analysis detects performance changes, + with a link to the dashboard filtered by the relevant configuration. -| Key | Prompt tokens | Output tokens | Rates | Max seconds | -|-----|--------------|---------------|-------|-------------| -| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | -| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | -| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | -| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | +Notifications are sent to the `psap-rhaiis-alerts` Slack channel. The post-cleanup +step also detects infrastructure failures that occur before the test step runs. -## Postprocessing and KPIs +## PyTorch profiling -After each benchmark run, the rhaiis postprocess plugin (`projects/rhaiis/postprocess/`) -automatically computes 22 KPIs with model-furnace parity: +When `rhaiis.profiler.enabled: true`, the pipeline runs profiler-gated benchmarks +before the main benchmarks: -| Category | KPIs | -|----------|------| -| Throughput | request_rate, tokens/s, input_tokens/s, output_tokens/s | -| Latency | ttft (median, p95, p99), itl (median, p95, p99), tpot (median, p95, p99) | -| Request latency | request_latency (median, p95) | -| Token efficiency | input_tokens/request, output_tokens/request | -| Request stats | concurrency, completed_requests, failed_requests | -| Benchmark | duration, prompt_token_count_mean | +1. Verify profiler prerequisites on the vLLM pod (gate file, sitecustomize.py) +2. Enable the profiler gate with a workload-specific label (e.g. `isl1000_osl1000`) +3. Run a GuideLLM benchmark at the configured profiler rates +4. Disable the profiler gate +5. Copy Chrome trace JSON files from the vLLM pod +6. Upload traces to S3 organized by accelerator/model/TP/version/profile -KPIs are written to `kpis.json` in the postprocess output directory. +Traces are viewable in `chrome://tracing` or Perfetto UI. -## PSAP payload +## Parallel job isolation -A model-furnace compatible `PSAP_perf____.json` -is generated after each benchmark. It wraps the raw `benchmarks.json` with experiment -metadata (model, vLLM image/args, accelerator, timestamps) for downstream compatibility. +Multiple FournosJobs can run concurrently on the same cluster. Each job gets a +unique deployment name by appending the last 5 characters of the FJOB name +(e.g. `nvidia-nemotron-3-super-120b-a1-54r5f`). Resource cleanup is scoped to +the current job's suffix to avoid interfering with other running jobs. ## MLflow export @@ -281,18 +469,48 @@ MLflow tags set on each run: | Tag | Example | |-----|---------| | `project` | rhaiis | -| `model_key` | qwen3-0_6b | -| `hf_model_id` | Qwen/Qwen3-0.6B | +| `model_key` | nemotron3super-120b-fp8 | +| `hf_model_id` | nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 | | `accelerator` | nvidia | -| `tensor_parallel_size` | 1 | -| `vllm_image` | quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2 | -| `vllm_version` | 3.4.0-ea.2-1773886296 | -| `workload_key` | profile1 | -| `rates` | 1,5,10 | +| `tensor_parallel_size` | 2 | +| `vllm_image` | vllm/vllm-openai:v0.24.0 | +| `vllm_version` | v0.24.0 | +| `workload_key` | profile1,profile2,profile4 | +| `rates` | 1,50,100,200,300 | | `guidellm_backend_type` | openai_http | MLflow experiment: `forge-rhaiis` +## Available models + +52 models are defined in `config.d/models.yaml`. Key families: + +| Family | Key examples | TP size | +|--------|-------------|---------| +| Llama-4 Scout | `llama-4-scout`, `llama-4-scout-fp8`, `llama-4-scout-int4` | 2-4 | +| Llama-4 Maverick | `llama-4-maverick`, `llama-4-maverick-fp8` | 8 | +| Llama-3.3-70B | `llama-3-3-70b`, `llama-3-3-70b-fp8`, `-w8a8`, `-w4a16` | 4 | +| Llama-3.1-8B | `llama-3-1-8b`, `llama-3-1-8b-fp8`, `-w8a8`, `-w4a16` | 1 | +| Llama-3.1-405B | `llama-3-1-405b`, `llama-3-1-405b-fp8`, `-w8a8` | 8 | +| Nemotron 120B | `nemotron3super-120b-fp8` | 2 | +| Granite 3.1 8B | `granite-3-1-8b-instruct`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Mistral Small 3.1 | `mistral-2503`, `-fp8`, `-w4a16`, `-w8a8` | 1 | +| Qwen3 235B | `qwen3-235b-instruct`, `-fp8` | 4 | +| DeepSeek | `deepseek-r1-0528`, `deepseek-v3-2`, `deepseek-v4-pro` | 8 | +| Phi-4 | `phi-4`, `phi-4-fp8`, `-w4a16`, `-w8a8` | 1 | +| Validation | `qwen3-0_6b` | 1 | + +Full list: `grep "^[a-z]" orchestration/config.d/models.yaml` + +## Workload profiles + +| Key | Prompt tokens | Output tokens | Rates | Max seconds | +|-----|--------------|---------------|-------|-------------| +| `profile1` | 1000 | 1000 | 1, 50, 100, 200, 300 | 450 | +| `profile2` | 512 (stdev 128) | 2048 (stdev 512) | 1, 50, 100, 200, 300 | 450 | +| `profile3` | 2048 | 128 | 1, 50, 100, 200, 300 | 450 | +| `profile4` | 8000 | 1000 | 1, 25, 50, 75, 100 | 450 | + ## Presets Presets in `presets.d/presets.yaml` provide shortcuts for common configurations: diff --git a/projects/rhaiis/orchestration/analysis.py b/projects/rhaiis/orchestration/analysis.py new file mode 100644 index 000000000..724b99b5f --- /dev/null +++ b/projects/rhaiis/orchestration/analysis.py @@ -0,0 +1,305 @@ +"""RHAIIS regression analysis and agent analysis helpers.""" + +from __future__ import annotations + +import logging +import os +import tempfile +from pathlib import Path + +from projects.core.library import env + +logger = logging.getLogger(__name__) + + +def run_standalone_analysis( + model_cfg: dict, + accelerator_key: str, + engine_args: dict, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, +) -> None: + """Run regression check + agent analysis using existing S3 data (no new benchmark).""" + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if not agent_cfg.get("enabled", False): + logger.info("Standalone analysis skipped: agent_analysis not enabled") + return + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not version or not compare_version: + logger.info("Standalone analysis skipped: version or compare_version not configured") + return + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping standalone analysis") + return + + accelerator = ( + accelerator_key.split("_")[0].upper() if "_" in accelerator_key else accelerator_key.upper() + ) + + consolidated_path = None + current_csv_path = None + try: + import pandas as pd + + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + + df = pd.read_csv(consolidated_path, on_bad_lines="warn") + for col in df.columns: + if df[col].dtype == object: + df[col] = df[col].str.strip() + + model_id = model_cfg.get("hf_model_id", "") + tp = str( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) + + current_rows = df[ + (df["version"] == version) + & (df["model"] == model_id) + & (df["accelerator"] == accelerator) + & (pd.to_numeric(df["TP"], errors="coerce").fillna(-1).astype(int).astype(str) == tp) + ] + + if current_rows.empty: + logger.warning( + "No data found in S3 for version=%s, model=%s, accelerator=%s, TP=%s", + version, + model_id, + accelerator, + tp, + ) + return + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + current_csv_path = tmp.name + current_rows.to_csv(tmp, index=False) + + logger.info("Standalone analysis: found %d rows for version=%s", len(current_rows), version) + run_regression_check( + current_csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid=run_uuid, + restrict_profiles=restrict_profiles, + engine_args=engine_args, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + if current_csv_path and os.path.exists(current_csv_path): + os.unlink(current_csv_path) + + +def run_regression_check( + csv_path, + compare_version: str, + current_version: str, + model_cfg: dict, + accelerator: str, + *, + run_uuid: str = "", + restrict_profiles: list[str] | None = None, + engine_args: dict | None = None, +) -> None: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + from projects.core.library import config + from projects.rhaiis.postprocess.regression import METRICS, PROFILE_MAP, run_regression_analysis + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = csv_dashboard_cfg.get("s3_key", "") + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + logger.warning("AWS credentials not available, skipping regression analysis") + return + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + s3.download_file(s3_bucket, s3_key, consolidated_path) + except Exception as e: + logger.warning("Could not download consolidated CSV for regression check: %s", e) + return + + try: + output_file = Path(env.ARTIFACT_DIR) / "regression_analysis.json" + analysis = run_regression_analysis( + current_csv_path=Path(str(csv_path)), + consolidated_csv_path=Path(consolidated_path), + compare_version=compare_version, + current_version=current_version, + output_file=output_file, + profile_map=PROFILE_MAP, + metrics=METRICS, + restrict_profiles=restrict_profiles, + ) + + if analysis.get("regression_count", 0) > 0 or analysis.get("improvement_count", 0) > 0: + report_url = "" + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + if agent_cfg.get("enabled", False): + report_url = run_agent_analysis( + analysis, + model_cfg, + accelerator, + current_version, + compare_version, + run_uuid, + severity_threshold=agent_cfg.get("severity_threshold", 10), + ) + + from projects.rhaiis.postprocess.regression import send_regression_notification + + _ea = engine_args or {} + tp = _ea.get("tensor-parallel-size") or _ea.get("tp-size") or _ea.get("tp_size") or "" + dp = _ea.get("data-parallel-size") or _ea.get("dp-size") or "" + send_regression_notification( + analysis, + model=model_cfg.get("hf_model_id", ""), + accelerator=accelerator, + job_id=run_uuid, + slack_user=config.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + report_url=report_url, + tp=str(tp), + dp=str(dp), + ) + except Exception: + logger.warning("Regression analysis failed; continuing", exc_info=True) + finally: + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def run_agent_analysis( + analysis: dict, + model_cfg: dict, + accelerator: str, + current_version: str, + compare_version: str, + run_uuid: str, + *, + severity_threshold: int = 10, +) -> str: + """Request AI agent analysis for severe regressions. Returns report URL or empty string.""" + from projects.core.library import config + from projects.rhaiis.postprocess.agent import ( + AGENT_SEVERITY_THRESHOLD, + build_pr_followup_prompt, + check_agent_connectivity, + markdown_to_html, + request_agent_analysis, + send_followup, + ) + + agent_cfg = config.project.get_config("rhaiis.agent_analysis", {}) + agent_url = agent_cfg.get("url", "") + if not agent_url: + logger.warning("Agent analysis enabled but no URL configured (rhaiis.agent_analysis.url)") + return "" + + threshold = severity_threshold or AGENT_SEVERITY_THRESHOLD + severe = [r for r in analysis.get("regressions", []) if abs(r["pct_diff"]) > threshold] + if not severe: + logger.info("No severe regressions (>%d%%), skipping agent analysis", threshold) + return "" + + ok, detail = check_agent_connectivity(agent_url) + if not ok: + logger.warning("Agent not reachable, skipping analysis: %s", detail) + return "" + + ea = model_cfg.get("engine_args", {}) + tp = str(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) + model = model_cfg.get("hf_model_id", "") + improvements = analysis.get("improvements", []) + + agent_response = request_agent_analysis( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe, + job_id=run_uuid, + improvements=improvements if improvements else None, + agent_url=agent_url, + ) + if not agent_response: + return "" + + pr_prompt = build_pr_followup_prompt(current_version, compare_version) + pr_analysis = send_followup(message=pr_prompt, job_id=run_uuid, agent_url=agent_url) + if pr_analysis: + agent_response = f"{agent_response}\n\n---\n\n## Related Pull Requests\n\n{pr_analysis}" + + html_content = markdown_to_html( + agent_response, + run_uuid, + model, + current_version, + compare_version, + ) + html_path = Path(env.ARTIFACT_DIR) / f"agent_analysis_{run_uuid}.html" + html_path.write_text(html_content, encoding="utf-8") + logger.info("Agent analysis saved to %s", html_path) + + try: + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + vault_name = s3_cfg.get("vault", "") + credentials_file = s3_cfg.get("credentials_file", "aws.credentials") + credentials_path = get_aws_credentials(vault_name, credentials_file) + if credentials_path: + s3 = create_s3_client(credentials_path) + s3_bucket = s3_cfg.get("bucket", "") + s3_key = f"reports/rhaiis/{run_uuid}_analysis.html" + s3.upload_file( + str(html_path), + s3_bucket, + s3_key, + ExtraArgs={"ContentType": "text/html"}, + ) + report_url = s3.generate_presigned_url( + "get_object", + Params={"Bucket": s3_bucket, "Key": s3_key}, + ExpiresIn=2592000, + ) + logger.info("Agent analysis uploaded to S3, presigned URL generated") + return report_url + except Exception: + logger.warning("Failed to upload agent analysis to S3; continuing", exc_info=True) + + return "" diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 6b50a96e2..89a72f62e 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import logging +import os import types +from pathlib import Path import click import prepare_rhaiis @@ -16,6 +18,72 @@ logger = logging.getLogger(__name__) +def _check_pipeline_failure_and_notify() -> None: + """Detect early pipeline failures (e.g. image pull errors) and send a Slack alert. + + Runs in the post-cleanup finally step. Checks whether prior steps + produced FAILURE artifacts or the test step was skipped entirely. + """ + try: + base_dir_env = os.environ.get("ARTIFACT_BASE_DIR", "") + if not base_dir_env: + return + base_dir = Path(base_dir_env) + if not base_dir.is_dir(): + return + + test_dir = base_dir / "03__test" + test_ran = test_dir.exists() + + # Skip steps already handled by do_test's exception handler + failure_files = sorted(f for f in base_dir.glob("*/FAILURE") if f.parent.name != "03__test") + + if not failure_files and test_ran: + return + + errors = [] + for f in failure_files: + step_name = f.parent.name + content = f.read_text().strip() + summary = content[:300] if content else "unknown error" + errors.append(f"[{step_name}] {summary}") + + if not test_ran and not errors: + errors.append( + "Test step was skipped — likely an earlier pipeline step failed (e.g. image pull timeout)" + ) + + error_text = "\n".join(errors) + + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_key = _cfg.project.get_config("tests.rhaiis.model_key", "unknown") + try: + model_cfg = runtime_config.get_model(model_key) + model_name = model_cfg.get("hf_model_id", model_key) + except Exception: + model_name = model_key + + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() + + send_failure_notification( + error=error_text, + model=model_name, + accelerator=accelerator_key, + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + version=_cfg.project.get_config("tests.rhaiis.version", ""), + cluster=cluster_tag, + ) + except Exception: + logger.warning("Failed to check/send pipeline failure notification", exc_info=True) + + def list_vaults() -> list[str]: test_rhaiis.init() return runtime_config.get_vaults() @@ -29,8 +97,8 @@ def resolve_hardware_request(hardware_spec: dict) -> dict: model_key = runtime_config.get_test_model_key() model = runtime_config.get_model(model_key) - vllm_args = model.get("vllm_args", {}) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) + ea = model.get("engine_args", {}) + tp_size = int(ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1) accelerator = runtime_config.get_accelerator() gpu_type = runtime_config.get_gpu_type(accelerator) @@ -85,6 +153,7 @@ def pre_cleanup(ctx): @ci_lib.safe_ci_entrypoint def post_cleanup(ctx): """Post-cleanup phase - Clean up resources after test.""" + _check_pipeline_failure_and_notify() return prepare_rhaiis.cleanup() diff --git a/projects/rhaiis/orchestration/cli.py b/projects/rhaiis/orchestration/cli.py index a6cc2e626..cea6215e6 100644 --- a/projects/rhaiis/orchestration/cli.py +++ b/projects/rhaiis/orchestration/cli.py @@ -51,7 +51,13 @@ def cli(ctx): @click.option("--namespace", "-n", default=None, help="Kubernetes namespace") @click.option("--deployment-name", default=None, help="Deployment name (defaults to model name)") @click.option("--accelerator", type=click.Choice(["nvidia", "amd"]), default=None) -@click.option("--vllm-image", help="vLLM container image override") +@click.option( + "--engine", + type=click.Choice(["vllm", "sglang", "trtllm"]), + default=None, + help="Inference engine", +) +@click.option("--serving-image", help="Serving engine container image override") @click.option("--tensor-parallel", "-tp", type=int, help="Tensor parallel size override") @click.option("--replicas", "-r", type=int, default=None) @click.option("--storage-source", type=click.Choice(["hf", "pvc"]), default=None) @@ -70,7 +76,8 @@ def test( namespace: str | None, deployment_name: str | None, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -93,7 +100,8 @@ def test( _apply_cli_overrides( workload_key=workload_key, accelerator=accelerator, - vllm_image=vllm_image, + engine=engine, + serving_image=serving_image, tensor_parallel=tensor_parallel, replicas=replicas, storage_source=storage_source, @@ -113,7 +121,7 @@ def test( try: test_phase.run( model_key=model_key, - workload_key=workload_key, + workload_keys=[workload_key], namespace=namespace, deployment_name=deployment_name, ) @@ -128,7 +136,8 @@ def _apply_cli_overrides( *, workload_key: str, accelerator: str | None, - vllm_image: str | None, + engine: str | None, + serving_image: str | None, tensor_parallel: int | None, replicas: int | None, storage_source: str | None, @@ -140,11 +149,21 @@ def _apply_cli_overrides( ) -> None: if accelerator: config.project.set_config("rhaiis.accelerator", accelerator) - if vllm_image: + if engine: + config.project.set_config("rhaiis.engine", engine) + resolved_engine = runtime_config.get_engine() + if serving_image: resolved_accel = runtime_config.get_accelerator() - config.project.set_config(f"rhaiis.images.{resolved_accel}", vllm_image) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.images.{resolved_accel}", serving_image + ) if tensor_parallel is not None: - config.project.set_config("rhaiis.vllm_args.tensor-parallel-size", tensor_parallel) + tp_key = {"sglang": "tp-size", "trtllm": "tp_size"}.get( + resolved_engine, "tensor-parallel-size" + ) + config.project.set_config( + f"rhaiis.engines.{resolved_engine}.args.{tp_key}", tensor_parallel + ) if replicas is not None: config.project.set_config("rhaiis.deploy.replicas", replicas) if storage_source: @@ -169,9 +188,10 @@ def _print_dry_run( workload_cfg = runtime_config.get_workload(workload_key) deploy_cfg = runtime_config.get_deploy_config() accelerator = runtime_config.get_accelerator() - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload_cfg) + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_args = runtime_config.merge_engine_args(engine_defaults, model_cfg, workload_cfg, engine) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) if not deployment_name: @@ -182,9 +202,10 @@ def _print_dry_run( click.echo(f" Workload: {workload_key}") click.echo(f" Namespace: {namespace}") click.echo(f" Deployment: {deployment_name}") + click.echo(f" Engine: {engine}") click.echo(f" Accelerator: {accelerator}") - click.echo(f" Image: {vllm_image}") - click.echo(f" vLLM args: {vllm_args}") + click.echo(f" Image: {serving_image}") + click.echo(f" Engine args: {engine_args}") click.echo(f" Env vars: {env_vars}") click.echo(f" Replicas: {deploy_cfg.get('replicas', 1)}") click.echo( diff --git a/projects/rhaiis/orchestration/config.d/models.yaml b/projects/rhaiis/orchestration/config.d/models.yaml index 806ce0442..1a839f919 100644 --- a/projects/rhaiis/orchestration/config.d/models.yaml +++ b/projects/rhaiis/orchestration/config.d/models.yaml @@ -2,46 +2,46 @@ qwen3-0_6b: name: Qwen3-0.6B hf_model_id: Qwen/Qwen3-0.6B - vllm_args: {} + engine_args: {} # Llama-4 Scout (MoE) llama-4-scout: name: Llama-4-Scout-17B-16E-Instruct hf_model_id: meta-llama/Llama-4-Scout-17B-16E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-4-scout-fp8: name: Llama-4-Scout-17B-16E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 2 kv-cache-dtype: fp8 llama-4-scout-int4: name: Llama-4-Scout-17B-16E-Instruct-INT4 hf_model_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 2 # Llama-4 Maverick (MoE) llama-4-maverick: name: Llama-4-Maverick-17B-128E-Instruct hf_model_id: meta-llama/Llama-4-Maverick-17B-128E-Instruct - vllm_args: + engine_args: tensor-parallel-size: 8 llama-4-maverick-fp8: name: Llama-4-Maverick-17B-128E-Instruct-FP8 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 8 kv-cache-dtype: fp8 llama-4-maverick-w4a16: name: Llama-4-Maverick-17B-128E-Instruct-W4A16 hf_model_id: RedHatAI/Llama-4-Maverick-17B-128E-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 enable-expert-parallel: true @@ -66,25 +66,25 @@ mistral-2503-fp8: llama-3-3-70b: name: Llama-3.3-70B-Instruct hf_model_id: meta-llama/Llama-3.3-70B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-fp8: name: Llama-3.3-70B-Instruct-FP8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w8a8: name: Llama-3.3-70B-Instruct-W8A8 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w8a8 - vllm_args: + engine_args: tensor-parallel-size: 4 llama-3-3-70b-w4a16: name: Llama-3.3-70B-Instruct-W4A16 hf_model_id: RedHatAI/Llama-3.3-70B-Instruct-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 4 # Granite 3.1 8B @@ -125,7 +125,7 @@ qwen25-7b-w4a16: qwen3-n-80b-a3b-instruct: name: Qwen3-Next-80B-A3B-Instruct hf_model_id: Qwen/Qwen3-Next-80B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-30b-a3b-instruct: @@ -135,13 +135,13 @@ qwen3-30b-a3b-instruct: qwen3-30b-a3b: name: Qwen3-30B-A3B hf_model_id: Qwen/Qwen3-30B-A3B - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-235b-instruct: name: Qwen3-235B-A22B-Instruct hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507 - vllm_args: + engine_args: tensor-parallel-size: 4 env_vars: VLLM_ROCM_USE_AITER: "0" @@ -149,7 +149,7 @@ qwen3-235b-instruct: qwen3-235b-instruct-fp8: name: Qwen3-235B-A22B-Instruct-FP8 hf_model_id: Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true kv-cache-dtype: fp8 @@ -157,20 +157,20 @@ qwen3-235b-instruct-fp8: qwen3-235b-fp8-dynamic: name: Qwen3-235B-A22B-FP8-dynamic hf_model_id: RedHatAI/Qwen3-235B-A22B-FP8-dynamic - vllm_args: + engine_args: tensor-parallel-size: 4 enable-expert-parallel: true qwen3-vl-235b-fp8: name: Qwen3-VL-235B-A22B-FP8 hf_model_id: Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 qwen3-vl-30b-a3b: name: Qwen3-VL-30B-A3B-Instruct hf_model_id: Qwen/Qwen3-VL-30B-A3B-Instruct - vllm_args: + engine_args: tensor-parallel-size: 4 # GPT-OSS @@ -200,7 +200,7 @@ mixtral-8x7b: nemotron-3-1-70b: name: Nemotron-70B-Instruct hf_model_id: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF - vllm_args: + engine_args: tensor-parallel-size: 2 nemotron-3-1-70b-fp8: @@ -224,25 +224,25 @@ gemma-9b-w8a16: deepseek-v4-pro: name: DeepSeek-V4-Pro hf_model_id: deepseek-ai/DeepSeek-V4-Pro - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528: name: DeepSeek-R1-0528 hf_model_id: deepseek-ai/DeepSeek-R1-0528 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-r1-0528-w4a16: name: DeepSeek-R1-0528-W4A16 hf_model_id: RedHatAI/DeepSeek-R1-0528-quantized.w4a16 - vllm_args: + engine_args: tensor-parallel-size: 8 deepseek-v3-2: name: DeepSeek-V3.2 hf_model_id: deepseek-ai/DeepSeek-V3.2 - vllm_args: + engine_args: tensor-parallel-size: 8 # Mistral 7B @@ -258,30 +258,30 @@ nemotron3nano-30b: nemotron3nano-30b-fp8: name: Nemotron-3-Nano-30B-A3B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 nemotron3super-120b-fp8: name: Nemotron-3-Super-120B-A12B-FP8 hf_model_id: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - vllm_args: + engine_args: tensor-parallel-size: 4 # Ministral ministral-3-14b-reasoning: name: Ministral-3-14B-Reasoning hf_model_id: mistralai/Ministral-3-14B-Reasoning-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp16: name: Ministral-3-14B-Instruct hf_model_id: mistralai/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 ministral-3-14b-fp8: name: Ministral-3-14B-FP8 hf_model_id: RedHatAI/Ministral-3-14B-Instruct-2512 - vllm_args: + engine_args: tensor-parallel-size: 4 diff --git a/projects/rhaiis/orchestration/config.d/rhaiis.yaml b/projects/rhaiis/orchestration/config.d/rhaiis.yaml index 4965747f1..89b904e48 100644 --- a/projects/rhaiis/orchestration/config.d/rhaiis.yaml +++ b/projects/rhaiis/orchestration/config.d/rhaiis.yaml @@ -1,5 +1,6 @@ -namespace: forge-rhaiis +namespace: kserve-e2e-perf accelerator: nvidia +cluster_tag: "" gpu_types: nvidia: h200 @@ -10,29 +11,87 @@ images: nvidia: quay.io/aipcc/rhaiis/cuda-ubi9:3.4.0-ea.2-1773886296 amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5-1766067105 +engine: vllm + +engines: + vllm: + images: + nvidia: vllm/vllm-openai:v0.24.0 + amd: quay.io/aipcc/rhaiis/rocm-ubi9:3.2.5 + port: 8080 + args: + gpu-memory-utilization: 0.92 + trust-remote-code: true + no-enable-log-requests: true + no-enable-prefix-caching: true + uvicorn-log-level: debug + sglang: + images: + nvidia: lmsysorg/sglang:v0.5.11 + amd: lmsysorg/sglang:v0.5.4.post2-rocm700-mi30x + port: 8080 + args: {} + trtllm: + images: + nvidia: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc13 + port: 8888 + args: {} + trtllm_config: + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.90 + mamba_ssm_cache_dtype: "" + mamba_ssm_stochastic_rounding: false + mamba_ssm_philox_rounds: 0 + cuda_graph_config: + enable_padding: false + max_batch_size: 256 + enable_attention_dp: false + enable_chunked_prefill: false + num_postprocess_workers: 0 + moe_config: + backend: CUTLASS + stream_interval: 0 + print_iter_log: false + deploy: replicas: 1 cpu_request: "4" memory_request: "16Gi" storage_source: hf - storage_pvc: model-pvc-2 - image_pull_secret: "" + storage_pvc: model-pvc + image_pull_secrets: [] service_account_name: "" ready_timeout: 3600 health_check_timeout: 120 -vllm_args: - gpu-memory-utilization: 0.92 - trust-remote-code: true - no-enable-log-requests: true - no-enable-prefix-caching: true - uvicorn-log-level: debug - max-model-len: 8192 - tensor-parallel-size: 1 - env_vars: {} accelerator_env_vars: nvidia: {} amd: VLLM_ROCM_USE_AITER: "1" + +s3: + bucket: psap-dashboard-data + vault: psap-forge-dashboard-s3 + credentials_file: aws.credentials + +warmup: + rate: 200 + max_seconds: 60 + +profiler: + enabled: false + s3_bucket: psap-dashboard-data + s3_prefix: pytorch-profiles/rhaiis + vault: psap-forge-dashboard-s3 + labels: [] + rates: [200] + max_seconds: 200 + +agent_analysis: + enabled: false + severity_threshold: 10 + url: "http://agent-gateway.psap-ai-analysis-agent-gateway.svc.cluster.local:8443/v1/stream" diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index 95231ac5a..1cee1e4c6 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -5,12 +5,12 @@ ci-quick: profile1: data: "prompt_tokens=1000,output_tokens=1000" - rates: [1, 50, 100, 200, 300] + rates: [1,50,100,200,300] max_seconds: 450 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" - rates: [1, 50, 100, 200, 300] + rates: [1,50,100,200,300] max_seconds: 450 profile3: @@ -20,8 +20,7 @@ profile3: profile4: data: "prompt_tokens=8000,output_tokens=1000" - rates: [1, 25, 50, 75, 100] + rates: [1,25,50,75,100] max_seconds: 450 samples: 50 - vllm_args: - max-model-len: 10000 + diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index 045227198..c158eb8eb 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -1,6 +1,8 @@ vaults: - psap-forge-notifications - psap-forge-mlflow-export +- psap-forge-aws-s3-export +- psap-forge-dashboard-s3 benchmarks: guidellm: @@ -9,6 +11,7 @@ benchmarks: image: ghcr.io/vllm-project/guidellm:v0.6.0 timeout: 14400 pvc_size: 10Gi + hf_token_secret: "storage-config" args: backend_type: openai_http rate_type: concurrent @@ -19,6 +22,12 @@ tests: rhaiis: model_key: qwen3-0_6b workload_key: profile1 + workload_keys: [] + run_benchmark: true + warmup: true + version: "" + compare_version: "" + slack_user: "" caliper: postprocess: @@ -31,11 +40,19 @@ caliper: enabled: false kpi: enabled: true - generate: + artifacts_to_kpis: enabled: true output: kpis.json - export: - enabled: false + kpis_to_csv: + enabled: true + output: dashboard.csv + csv_dashboard: + enabled: true + s3_key: staging/rhaiis-dashboard/consolidated_dashboard.csv + analyze: + enabled: false + historical_kpis: historical_kpis + output: regression_analysis.json export: from: null diff --git a/projects/rhaiis/orchestration/manifests.py b/projects/rhaiis/orchestration/manifests.py new file mode 100644 index 000000000..caa96c821 --- /dev/null +++ b/projects/rhaiis/orchestration/manifests.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def build_servingruntime( + *, + deployment_name: str, + namespace: str, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + image_pull_secrets: list[str] | None = None, + env_vars: dict | None = None, + trtllm_config: dict | None = None, +) -> dict[str, Any]: + """Build a KServe ServingRuntime manifest dict.""" + env_vars_list = [{"name": k, "value": str(v)} for k, v in (env_vars or {}).items()] + + manifest: dict[str, Any] = { + "apiVersion": "serving.kserve.io/v1alpha1", + "kind": "ServingRuntime", + "metadata": { + "annotations": { + "opendatahub.io/template-display-name": f"ServingRuntime for {engine} | RHAIIS", + }, + "labels": {"opendatahub.io/dashboard": "true"}, + "name": deployment_name, + "namespace": namespace, + }, + "spec": { + "builtInAdapter": {"modelLoadingTimeoutMillis": 300000}, + "multiModel": False, + "supportedModelFormats": [ + {"autoSelect": True, "name": _model_format_name(engine)}, + ], + }, + } + + if image_pull_secrets: + manifest["spec"]["imagePullSecrets"] = [{"name": s} for s in image_pull_secrets] + + if engine == "trtllm": + container = _build_trtllm_container( + model_id=model_id, + serving_image=serving_image, + engine_args=engine_args, + engine_port=engine_port, + env_vars_list=env_vars_list, + trtllm_config=trtllm_config, + ) + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "128Gi"}}, + ] + else: + container = _build_vllm_sglang_container( + model_id=model_id, + serving_image=serving_image, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, + storage_source=storage_source, + gpu_count=gpu_count, + env_vars_list=env_vars_list, + ) + if gpu_count > 1: + manifest["spec"]["volumes"] = [ + {"name": "shared-memory", "emptyDir": {"medium": "Memory", "sizeLimit": "64Gi"}}, + ] + + manifest["spec"]["containers"] = [container] + return manifest + + +def build_inferenceservice( + *, + deployment_name: str, + namespace: str, + engine: str, + engine_port: int, + accelerator: str, + gpu_count: int, + replicas: int, + cpu_request: str, + memory_request: str, + storage_source: str, + storage_pvc: str, + model_id: str, + service_account_name: str = "", + labels: dict | None = None, +) -> dict[str, Any]: + """Build a KServe InferenceService manifest dict.""" + annotations: dict[str, str] = { + "serving.kserve.io/deploymentMode": "RawDeployment", + "storage.kserve.io/readonly": "false", + } + if engine != "sglang": + annotations.update( + { + "serving.kserve.io/enable-prometheus-scraping": "true", + "prometheus.io/scrape": "true", + "prometheus.io/path": "/metrics", + "prometheus.io/port": str(engine_port), + } + ) + + metadata: dict[str, Any] = { + "annotations": annotations, + "name": deployment_name, + "namespace": namespace, + } + + if labels: + meta_labels = dict(labels) + if engine in ("sglang", "trtllm"): + meta_labels["monitoring.opendatahub.io/scrape"] = "false" + metadata["labels"] = meta_labels + + resources = _build_resources( + accelerator=accelerator, + gpu_count=gpu_count, + cpu_request=cpu_request, + memory_request=memory_request, + engine=engine, + ) + + model_spec: dict[str, Any] = { + "modelFormat": {"name": _model_format_name(engine)}, + "runtime": deployment_name, + "resources": resources, + } + + if storage_source == "hf" and storage_pvc: + model_spec["storageUri"] = f"pvc://{storage_pvc}/" + elif storage_source != "hf": + model_spec["storageUri"] = f"{storage_source}://{model_id}" + + predictor: dict[str, Any] = { + "minReplicas": replicas, + "maxReplicas": replicas, + "model": model_spec, + } + + if engine == "trtllm": + predictor["terminationGracePeriodSeconds"] = 120 + + if service_account_name: + predictor["serviceAccountName"] = service_account_name + + return { + "apiVersion": "serving.kserve.io/v1beta1", + "kind": "InferenceService", + "metadata": metadata, + "spec": {"predictor": predictor}, + } + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _model_format_name(engine: str) -> str: + if engine == "sglang": + return "sglang" + if engine == "trtllm": + return "trtllm" + return "vLLM" + + +def _build_resources( + *, + accelerator: str, + gpu_count: int, + cpu_request: str, + memory_request: str, + engine: str, +) -> dict[str, Any]: + resources: dict[str, Any] = {} + gpu_key = "nvidia.com/gpu" if accelerator == "nvidia" else "amd.com/gpu" + + if accelerator in ("nvidia", "amd"): + limits: dict[str, str] = {gpu_key: str(gpu_count)} + requests: dict[str, str] = {gpu_key: str(gpu_count)} + + if accelerator == "nvidia" and engine == "trtllm": + limits["memory"] = memory_request + limits["ephemeral-storage"] = "50Gi" + requests["cpu"] = cpu_request + requests["memory"] = memory_request + requests["ephemeral-storage"] = "50Gi" + + resources["limits"] = limits + resources["requests"] = requests + + if not (accelerator == "nvidia" and engine == "trtllm"): + resources.setdefault("requests", {}) + resources["requests"]["cpu"] = cpu_request + resources["requests"]["memory"] = memory_request + + return resources + + +def _build_vllm_sglang_container( + *, + model_id: str, + serving_image: str, + engine: str, + engine_args: dict, + engine_port: int, + storage_source: str, + gpu_count: int, + env_vars_list: list[dict], +) -> dict[str, Any]: + if engine == "sglang": + command = ["sglang", "serve"] + if storage_source == "hf": + args = [f"--model-path={model_id}", "--port=8080", "--host=0.0.0.0"] + else: + args = ["--model-path=/mnt/models", "--port=8080", "--host=0.0.0.0"] + else: + command = ["python3", "-m", "vllm.entrypoints.openai.api_server"] + if storage_source == "hf": + args = [f"--model={model_id}", "--port=8080"] + else: + args = ["--model=/mnt/models", f"--served-model-name={model_id}", "--port=8080"] + + for key, val in (engine_args or {}).items(): + if isinstance(val, bool): + if val: + args.append(f"--{key}") + else: + args.append(f"--{key}={val}") + + env: list[dict[str, Any]] = [ + {"name": "USER", "value": engine}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "XDG_CONFIG_HOME", "value": "/tmp/.config"}, + ] + + if gpu_count > 1: + env.append({"name": "NCCL_DEBUG", "value": "WARN"}) + + if storage_source == "hf": + env.extend( + [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HOME", "value": "/mnt/models"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + ] + ) + if engine != "sglang": + env.append({"name": "VLLM_CACHE_DIR", "value": "/mnt/models/.cache/vllm"}) + env.extend( + [ + {"name": "HF_DATASETS_CACHE", "value": "/mnt/models/.cache/huggingface/datasets"}, + { + "name": "HF_TOKEN", + "valueFrom": { + "secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}, + }, + }, + ] + ) + else: + env.append({"name": "HOME", "value": "/tmp"}) + + env.extend(env_vars_list) + + container: dict[str, Any] = { + "name": "kserve-container", + "image": serving_image, + "command": command, + "args": args, + "env": env, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + } + + if gpu_count > 1: + container["volumeMounts"] = [{"name": "shared-memory", "mountPath": "/dev/shm"}] + + return container + + +def _build_trtllm_launch_script( + model_id: str, + engine_port: int, + engine_args: dict, + trtllm_config: dict | None, +) -> str: + safe_model = model_id.replace("/", "--") + parts = [ + "set -ex", + f'if [ ! -d "/mnt/models/hub/models--{safe_model}/snapshots" ]; then', + f" hf download {model_id}", + "fi", + 'python3 -c "', + "from flashinfer.norm import get_norm_module", + "get_norm_module()", + "print('FlashInfer norm kernel pre-compiled successfully')", + '"', + ] + + if trtllm_config: + import yaml + + config_yaml = yaml.dump(trtllm_config, default_flow_style=False).rstrip() + parts.extend(["cat > /tmp/trtllm-config.yml << EOF", config_yaml, "EOF"]) + + cmd = [f"trtllm-serve {model_id} \\", " --host 0.0.0.0 \\", f" --port={engine_port}"] + + for key, val in (engine_args or {}).items(): + if key == "trtllm-config": + continue + if isinstance(val, bool): + if val: + cmd.append(f" --{key}") + else: + cmd.append(f" --{key}={val}") + + if trtllm_config: + cmd.append(" --extra_llm_api_options=/tmp/trtllm-config.yml") + + # Add line continuations between cmd parts (except last) + cmd_lines = [] + for i, line in enumerate(cmd): + if i < len(cmd) - 1 and not line.endswith("\\"): + cmd_lines.append(f"{line} \\") + else: + cmd_lines.append(line) + + parts.extend(cmd_lines) + return "\n".join(parts) + "\n" + + +def _build_trtllm_container( + *, + model_id: str, + serving_image: str, + engine_args: dict, + engine_port: int, + env_vars_list: list[dict], + trtllm_config: dict | None, +) -> dict[str, Any]: + script = _build_trtllm_launch_script(model_id, engine_port, engine_args, trtllm_config) + + env: list[dict[str, Any]] = [ + {"name": "HF_HUB_OFFLINE", "value": "0"}, + {"name": "HF_HOME", "value": "/mnt/models"}, + { + "name": "HF_TOKEN", + "valueFrom": {"secretKeyRef": {"name": "storage-config", "key": "HF_TOKEN"}}, + }, + {"name": "TRTLLM_ENABLE_PDL", "value": "1"}, + {"name": "HOME", "value": "/tmp"}, + {"name": "XDG_CACHE_HOME", "value": "/tmp/.cache"}, + {"name": "FLASHINFER_WORKSPACE_BASE", "value": "/mnt/models"}, + ] + env.extend(env_vars_list) + + return { + "name": "kserve-container", + "image": serving_image, + "command": ["/bin/bash", "-c", script], + "env": env, + "securityContext": {"runAsUser": 0}, + "ports": [{"containerPort": engine_port, "protocol": "TCP"}], + "volumeMounts": [{"name": "shared-memory", "mountPath": "/dev/shm"}], + } diff --git a/projects/rhaiis/orchestration/notifications.py b/projects/rhaiis/orchestration/notifications.py new file mode 100644 index 000000000..fcaddc5f6 --- /dev/null +++ b/projects/rhaiis/orchestration/notifications.py @@ -0,0 +1,84 @@ +"""RHAIIS pipeline notification helpers.""" + +from __future__ import annotations + +import logging +import os +import traceback + +from projects.rhaiis.orchestration import runtime_config + +logger = logging.getLogger(__name__) + + +def _send_alert( + error_message: str, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Shared implementation for pipeline failure/warning Slack alerts.""" + from projects.core.library import config as _cfg + from projects.rhaiis.postprocess.regression import send_failure_notification + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + engine = runtime_config.get_engine() + engine_args = runtime_config.merge_engine_args( + runtime_config.get_engine_args(engine), + model_cfg, + runtime_config.get_workload(workload_keys[0]), + engine, + ) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or "" + ) + dp = engine_args.get("data-parallel-size") or engine_args.get("dp-size") or "" + + send_failure_notification( + error=error_message, + model=model_cfg.get("hf_model_id", model_key), + accelerator=f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper(), + job_id=os.environ.get("FJOB_NAME", ""), + slack_user=_cfg.project.get_config("tests.rhaiis.slack_user", ""), + notification_vault="psap-forge-notifications", + tp=str(tp), + dp=str(dp), + version=_cfg.project.get_config("tests.rhaiis.version", ""), + workload_keys=workload_keys, + cluster=cluster_tag, + ) + + +def send_pipeline_failure_alert( + exc: Exception, + *, + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert when the pipeline fails.""" + try: + error_message = traceback.format_exception_only(type(exc), exc)[-1].strip() + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) + except Exception: + logger.warning("Failed to send pipeline failure alert", exc_info=True) + + +def send_pipeline_warning( + *, + warnings: list[str], + model_key: str, + workload_keys: list[str], +) -> None: + """Best-effort Slack alert for non-critical pipeline warnings.""" + try: + warning_text = "\n".join(f"• {w}" for w in warnings) + error_message = f"Pipeline completed with warnings:\n{warning_text}" + _send_alert(error_message, model_key=model_key, workload_keys=workload_keys) + except Exception: + logger.warning("Failed to send pipeline warning alert", exc_info=True) diff --git a/projects/rhaiis/orchestration/prepare_rhaiis.py b/projects/rhaiis/orchestration/prepare_rhaiis.py index aa8c3b06a..ca182f4f3 100644 --- a/projects/rhaiis/orchestration/prepare_rhaiis.py +++ b/projects/rhaiis/orchestration/prepare_rhaiis.py @@ -1,4 +1,5 @@ import logging +import os from projects.core.dsl.utils.k8s import oc from projects.rhaiis.orchestration import runtime_config @@ -44,10 +45,30 @@ def prepare(): ) +def _delete_resources_by_suffix(resource_type: str, ns: str, suffix: str) -> None: + """Delete resources whose names end with the FJOB suffix.""" + result = oc("get", resource_type, "-n", ns, "-o", "name", check=False, log_stdout=False) + if result.returncode != 0 or not result.stdout: + return + for line in result.stdout.strip().splitlines(): + name = line.strip() + if name.endswith(f"-{suffix}"): + oc("delete", name, "-n", ns, "--ignore-not-found", check=False) + + def cleanup(): ns = runtime_config.get_namespace() logger.info(f"Cleaning up rhaiis benchmark resources in {ns}") - oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) - oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + fjob = os.environ.get("FJOB_NAME", "") + if fjob: + suffix = fjob.rsplit("-", 1)[-1] + for resource_type in ("inferenceservice", "servingruntime", "job", "pod", "pvc"): + _delete_resources_by_suffix(resource_type, ns, suffix) + else: + oc("delete", "inferenceservice", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "servingruntime", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "job", "--all", "-n", ns, "--ignore-not-found", check=False) + oc("delete", "pod", "--all", "-n", ns, "--ignore-not-found", check=False) + logger.info("Cleanup complete") diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index 86c620f6b..a2ed40b80 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -75,3 +75,13 @@ nvidia: amd: rhaiis.accelerator: amd + +# Engine +vllm: + rhaiis.engine: vllm + +sglang: + rhaiis.engine: sglang + +trtllm: + rhaiis.engine: trtllm diff --git a/projects/rhaiis/orchestration/runtime_config.py b/projects/rhaiis/orchestration/runtime_config.py index d17c4c7d0..3f898fa81 100644 --- a/projects/rhaiis/orchestration/runtime_config.py +++ b/projects/rhaiis/orchestration/runtime_config.py @@ -39,12 +39,30 @@ def get_benchmark_config() -> dict: return dict(config.project.get_config("benchmarks.guidellm")) -def get_vllm_image(accelerator: str) -> str: - return config.project.get_config(f"rhaiis.images.{accelerator}") +def get_engine() -> str: + return config.project.get_config("rhaiis.engine", "vllm") -def get_vllm_defaults() -> dict: - return dict(config.project.get_config("rhaiis.vllm_args")) +def get_serving_image(accelerator: str, engine: str | None = None) -> str: + engine = engine or get_engine() + try: + return config.project.get_config(f"rhaiis.engines.{engine}.images.{accelerator}") + except Exception: + return config.project.get_config(f"rhaiis.images.{accelerator}") + + +def get_engine_args(engine: str | None = None) -> dict: + engine = engine or get_engine() + return dict(config.project.get_config(f"rhaiis.engines.{engine}.args", {})) + + +def get_engine_port(engine: str | None = None) -> int: + engine = engine or get_engine() + return int(config.project.get_config(f"rhaiis.engines.{engine}.port", 8080)) + + +def get_trtllm_config() -> dict: + return dict(config.project.get_config("rhaiis.engines.trtllm.trtllm_config", {})) def get_model(model_key: str) -> dict: @@ -67,15 +85,66 @@ def get_test_workload_key() -> str: return config.project.get_config("tests.rhaiis.workload_key") -def merge_vllm_args( - defaults: dict, +def get_profiler_config() -> dict: + return dict(config.project.get_config("rhaiis.profiler", {}) or {}) + + +_COMMON_ARG_TRANSLATIONS: dict[str, dict[str, str]] = { + "sglang": { + "tensor-parallel-size": "tp-size", + "data-parallel-size": "dp-size", + }, + "trtllm": { + "tensor-parallel-size": "tp_size", + "data-parallel-size": "dp_size", + }, +} + + +def _translate_args(args: dict, engine: str) -> dict: + """Translate vLLM-style engine_args to another engine's arg naming. + + Only shared args (TP, DP) are translated; vLLM-specific args are dropped. + """ + mapping = _COMMON_ARG_TRANSLATIONS.get(engine, {}) + translated = {} + for key, val in args.items(): + if key in mapping: + translated[mapping[key]] = val + # Drop vLLM-only args that have no equivalent + return translated + + +def merge_engine_args( + overrides: dict, model: dict, workload: dict, + engine: str | None = None, ) -> dict: - merged = dict(defaults) - merged.update(model.get("vllm_args", {})) - merged.update(workload.get("vllm_args", {})) - return merged + engine = engine or get_engine() + engine_key = f"{engine}_args" + + # Use engine-specific block if present, otherwise fall back to engine_args + # and auto-translate common args when the engine isn't vLLM + model_args = model.get(engine_key) if engine != "vllm" else None + if model_args is None: + base = dict(model.get("engine_args", {})) + if engine != "vllm": + base = _translate_args(base, engine) + else: + base = dict(model_args) + + wl_args = workload.get(engine_key) if engine != "vllm" else None + if wl_args is None: + wl = dict(workload.get("engine_args", {})) + if engine != "vllm": + wl = _translate_args(wl, engine) + else: + wl = dict(wl_args) + + base.update(wl) + base.update(overrides) + return base def merge_env_vars(accelerator: str, model: dict) -> dict: diff --git a/projects/rhaiis/orchestration/test_phase.py b/projects/rhaiis/orchestration/test_phase.py index 762dbc2b1..246aa4195 100644 --- a/projects/rhaiis/orchestration/test_phase.py +++ b/projects/rhaiis/orchestration/test_phase.py @@ -1,6 +1,12 @@ from __future__ import annotations +import json import logging +import os +import uuid as _uuid_mod +from pathlib import Path + +import yaml from projects.core.library import env from projects.core.library.postprocess import run_and_postprocess, write_test_labels @@ -8,98 +14,235 @@ logger = logging.getLogger(__name__) +_K8S_NAME_MAX = 63 +_warnings: list[str] = [] + + +def _write_manifest(manifest: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + yaml.dump(manifest, f, default_flow_style=False, sort_keys=False) + + +def _guidellm_job_name(prefix: str, workload_key: str, deployment_name: str) -> str: + """Build a K8s-safe job name: {prefix}-{workload_key}-{model}, trimming model to fit.""" + base = f"{prefix}-{workload_key}-" + available = _K8S_NAME_MAX - len(base) + model = deployment_name[:available] if available > 0 else "" + return f"{base}{model}".rstrip("-") + def run( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: - return run_and_postprocess( + ret = run_and_postprocess( do_test, model_key=model_key, - workload_key=workload_key, + workload_keys=workload_keys, namespace=namespace, deployment_name=deployment_name, ) + try: + _sync_postprocessed_dashboard_csv(model_key, workload_keys) + except Exception: + logger.exception("Dashboard CSV S3 sync after postprocessing failed") + ret = 1 + + return ret + def do_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: - with env.NextArtifactDir("testing"): - return _run_test( - model_key=model_key, - workload_key=workload_key, - namespace=namespace, - deployment_name=deployment_name, - ) + try: + dir_name = f"test_{model_key}_{'_'.join(workload_keys)}" + with env.NextArtifactDir(dir_name): + return _run_test( + model_key=model_key, + workload_keys=workload_keys, + namespace=namespace, + deployment_name=deployment_name, + ) + except Exception as exc: + from projects.rhaiis.orchestration.notifications import send_pipeline_failure_alert + + send_pipeline_failure_alert(exc, model_key=model_key, workload_keys=workload_keys) + raise def _run_test( *, model_key: str, - workload_key: str, + workload_keys: list[str], namespace: str, deployment_name: str | None = None, ) -> int: + _warnings.clear() model_cfg = runtime_config.get_model(model_key) - workload = runtime_config.get_workload(workload_key) accelerator = runtime_config.get_accelerator() + gpu_type = runtime_config.get_gpu_type(accelerator) or accelerator + from projects.core.library import config as _cfg + + cluster_tag = _cfg.project.get_config("rhaiis.cluster_tag", "") + accelerator_key = f"{gpu_type}_{cluster_tag}".upper() if cluster_tag else gpu_type.upper() deploy_cfg = runtime_config.get_deploy_config() benchmark_cfg = runtime_config.get_benchmark_config() if not deployment_name: - deployment_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) - - vllm_image = runtime_config.get_vllm_image(accelerator) - vllm_defaults = runtime_config.get_vllm_defaults() - vllm_args = runtime_config.merge_vllm_args(vllm_defaults, model_cfg, workload) + base_name = runtime_config.derive_deployment_name(model_cfg["hf_model_id"]) + fjob = os.environ.get("FJOB_NAME", "") + suffix = fjob.rsplit("-", 1)[-1] if fjob else "" + if suffix: + # KServe hostname: {name}-predictor-{namespace} must be <= 63 chars + ns = runtime_config.get_namespace() + max_base = 63 - len(f"-predictor-{ns}") - len(suffix) - 1 + truncated_base = base_name[:max_base].rstrip("-") + deployment_name = f"{truncated_base}-{suffix}" + else: + deployment_name = base_name + + engine = runtime_config.get_engine() + serving_image = runtime_config.get_serving_image(accelerator, engine) + engine_defaults = runtime_config.get_engine_args(engine) + engine_port = runtime_config.get_engine_port(engine) + first_workload = runtime_config.get_workload(workload_keys[0]) + engine_args = runtime_config.merge_engine_args( + engine_defaults, model_cfg, first_workload, engine + ) env_vars = runtime_config.merge_env_vars(accelerator, model_cfg) - rates = workload.get("rates", [1]) - max_seconds = workload.get("max_seconds", 180) + from projects.core.library import config as _cfg - _create_test_labels(model_key, workload_key, accelerator, vllm_args) + version = _cfg.project.get_config("tests.rhaiis.version", "") + run_uuid = _cfg.project.get_config("tests.rhaiis.run_uuid", "") or str(_uuid_mod.uuid4()) + logger.info("Run UUID for this job: %s", run_uuid) - logger.info( - "Testing model=%s workload=%s accelerator=%s", model_cfg["name"], workload_key, accelerator - ) + import subprocess - from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( - run as run_guidellm_benchmark, - ) from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( wait_guidellm_benchmark_task, ) + from projects.rhaiis.orchestration.manifests import ( + build_inferenceservice, + build_servingruntime, + ) from projects.rhaiis.toolbox.deploy_kserve_isvc.main import run as deploy_kserve_isvc from projects.rhaiis.toolbox.wait_isvc_ready.main import run as wait_isvc_ready + fjob_name = os.environ.get("FJOB_NAME", "") + fjob_ns = os.environ.get("FOURNOS_WORKLOAD_NAMESPACE", "psap-automation") + if fjob_name: + try: + mgmt_env = {k: v for k, v in os.environ.items() if k != "KUBECONFIG"} + subprocess.run( + [ + "oc", + "annotate", + "fournosjob", + fjob_name, + "-n", + fjob_ns, + f"rhaiis.run-uuid={run_uuid}", + "--overwrite", + ], + check=False, + capture_output=True, + timeout=10, + env=mgmt_env, + ) + logger.info("Annotated FournosJob %s with run-uuid=%s", fjob_name, run_uuid) + except Exception: + logger.warning("Failed to annotate FournosJob with UUID", exc_info=True) + + from projects.core.library import config + + profiler_cfg = runtime_config.get_profiler_config() + profiler_enabled = profiler_cfg.get("enabled", False) + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + + # Standalone analysis only — no deployment needed + if not run_benchmark and not profiler_enabled: + logger.info("run_benchmark=false and profiler=false, running standalone analysis only") + try: + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + + run_standalone_analysis( + model_cfg, + accelerator_key, + engine_args, + run_uuid=run_uuid, + restrict_profiles=workload_keys or None, + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append("Standalone analysis failed") + return 1 if _warnings else 0 + benchmark_timeout = benchmark_cfg.get("timeout", 14400) wait_guidellm_benchmark_task._retry_config["attempts"] = max(1, benchmark_timeout // 10) try: + isvc_labels = {"opendatahub.io/dashboard": "true"} + if profiler_enabled and engine == "vllm": + isvc_labels["vllm-profiler/enabled"] = "true" + elif profiler_enabled and engine != "vllm": + logger.warning("Profiler is only supported with vLLM engine, skipping profiler") + profiler_enabled = False + logger.info("Deploying %s to %s/%s", model_cfg["hf_model_id"], namespace, deployment_name) - deploy_kserve_isvc( + ea = engine_args or {} + gpu_count = int( + ea.get("tensor-parallel-size") or ea.get("tp-size") or ea.get("tp_size") or 1 + ) + + sr_manifest = build_servingruntime( deployment_name=deployment_name, namespace=namespace, model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, - accelerator=accelerator, - vllm_args=vllm_args, + serving_image=serving_image, + engine=engine, + engine_args=engine_args, + engine_port=engine_port, + storage_source=deploy_cfg.get("storage_source", "hf"), + gpu_count=gpu_count, + image_pull_secrets=deploy_cfg.get("image_pull_secrets") or [], env_vars=env_vars, + trtllm_config=runtime_config.get_trtllm_config() if engine == "trtllm" else None, + ) + isvc_manifest = build_inferenceservice( + deployment_name=deployment_name, + namespace=namespace, + engine=engine, + engine_port=engine_port, + accelerator=accelerator, + gpu_count=gpu_count, replicas=deploy_cfg.get("replicas", 1), cpu_request=deploy_cfg.get("cpu_request", "4"), memory_request=deploy_cfg.get("memory_request", "16Gi"), storage_source=deploy_cfg.get("storage_source", "hf"), storage_pvc=deploy_cfg.get("storage_pvc", ""), - image_pull_secret=deploy_cfg.get("image_pull_secret", ""), + model_id=model_cfg["hf_model_id"], service_account_name=deploy_cfg.get("service_account_name", ""), + labels=isvc_labels, + ) + sr_file = env.ARTIFACT_DIR / "src" / "servingruntime.yaml" + isvc_file = env.ARTIFACT_DIR / "src" / "inferenceservice.yaml" + _write_manifest(sr_manifest, sr_file) + _write_manifest(isvc_manifest, isvc_file) + + deploy_kserve_isvc( + namespace=namespace, + servingruntime_file=str(sr_file), + inferenceservice_file=str(isvc_file), ) logger.info("Waiting for InferenceService to be ready") @@ -110,65 +253,250 @@ def _run_test( health_check_timeout=deploy_cfg.get("health_check_timeout", 120), ) - endpoint_url = f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:8080" - - logger.info("Running benchmark at rates=%s", rates) - - benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + endpoint_url = ( + f"http://{deployment_name}-predictor.{namespace}.svc.cluster.local:{engine_port}" + ) - guidellm_args = runtime_config.build_guidellm_args( - benchmark_cfg=benchmark_cfg, - model_id=model_cfg["hf_model_id"], - data=workload["data"], - rates=rates, - max_seconds=max_seconds, + warmup_enabled = ( + config.project.get_config("tests.rhaiis.warmup", True) + and run_benchmark + and not profiler_enabled ) - run_guidellm_benchmark( - endpoint_url=f"{endpoint_url}/v1", - name=f"guidellm-{deployment_name}", - namespace=namespace, - image=benchmark_image, - timeout=benchmark_timeout, - pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), - guidellm_args=guidellm_args, + logger.info( + "Running %d workload(s): %s", + len(workload_keys), + workload_keys, ) + + # Phase 1: warmup or profiler for ALL workloads first + for wl_key in workload_keys: + step_kwargs = dict( + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_cfg=benchmark_cfg, + model_cfg=model_cfg, + workload=runtime_config.get_workload(wl_key), + workload_key=wl_key, + benchmark_timeout=benchmark_timeout, + ) + if profiler_enabled: + logger.info("Running profiler for workload=%s", wl_key) + _run_profiler_step(**step_kwargs) + elif warmup_enabled: + logger.info("Running warmup for workload=%s", wl_key) + _run_warmup_step(**step_kwargs) + + if profiler_enabled: + try: + _upload_profiler_traces(model_cfg, gpu_type, engine_args, profiler_cfg) + except Exception: + logger.exception("Profiler trace upload failed") + _warnings.append("Profiler trace upload failed") + + # Phase 2: benchmark + post-processing for ALL workloads + trtllm_cfg = runtime_config.get_trtllm_config() if engine == "trtllm" else None + for wl_key in workload_keys: + _run_workload_benchmark( + model_key=model_key, + workload_key=wl_key, + model_cfg=model_cfg, + accelerator=accelerator, + accelerator_key=accelerator_key, + gpu_type=gpu_type, + serving_image=serving_image, + engine_args=engine_args, + benchmark_cfg=benchmark_cfg, + deployment_name=deployment_name, + namespace=namespace, + endpoint_url=endpoint_url, + benchmark_timeout=benchmark_timeout, + run_uuid=run_uuid, + version=version, + cluster_tag=cluster_tag, + trtllm_config=trtllm_cfg, + ) + + try: + first_workload = runtime_config.get_workload(workload_keys[0]) + first_rates = first_workload.get("rates", [1]) + first_max_seconds = first_workload.get("max_seconds", 180) + _set_mlflow_metadata( + model_key, + ",".join(workload_keys), + model_cfg, + accelerator, + serving_image, + engine_args, + benchmark_cfg, + first_rates, + first_max_seconds, + namespace, + deployment_name, + ) + except Exception: + logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) finally: _capture_and_cleanup(deployment_name, namespace) try: - _generate_psap_payload(model_cfg, accelerator, vllm_image, vllm_args, workload_key) + _upload_predictor_log(run_uuid) except Exception: - logger.warning("PSAP payload generation failed; continuing", exc_info=True) + logger.warning("Predictor log upload failed", exc_info=True) + _warnings.append("Predictor log upload failed") - try: - _set_mlflow_metadata( + if _warnings: + logger.warning( + "Test completed with %d warning(s): %s", len(_warnings), "; ".join(_warnings) + ) + from projects.rhaiis.orchestration.notifications import send_pipeline_warning + + send_pipeline_warning( + warnings=list(_warnings), + model_key=model_key, + workload_keys=workload_keys, + ) + _warnings.clear() + return 1 + + return 0 + + +def _run_workload_benchmark( + *, + model_key: str, + workload_key: str, + model_cfg: dict, + accelerator: str, + accelerator_key: str, + gpu_type: str, + serving_image: str, + engine_args: dict, + benchmark_cfg: dict, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_timeout: int, + run_uuid: str, + version: str, + cluster_tag: str, + trtllm_config: dict | None = None, +) -> None: + """Run benchmark and post-processing for a single workload. + + Each workload gets its own NextArtifactDir with per-workload test labels so + the Caliper parser treats them as separate test nodes and produces one + dashboard CSV row-set per profile. + """ + logger.info("=== Benchmark %s (UUID: %s) ===", workload_key, run_uuid) + + workload = runtime_config.get_workload(workload_key) + rates = workload.get("rates", [1]) + max_seconds = workload.get("max_seconds", 180) + + from projects.core.library import config + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) + + run_benchmark = config.project.get_config("tests.rhaiis.run_benchmark", True) + + with env.NextArtifactDir(f"benchmark_{workload_key}"): + _create_test_labels( model_key, workload_key, - model_cfg, accelerator, - vllm_image, - vllm_args, - benchmark_cfg, - rates, - max_seconds, - namespace, - deployment_name, + engine_args, + hf_model_id=model_cfg["hf_model_id"], + version=version, + serving_image=serving_image, + cluster_tag=cluster_tag, + accelerator_chip=gpu_type.upper(), + run_uuid=run_uuid, + trtllm_config=trtllm_config, ) - except Exception: - logger.warning("Setting MLflow metadata failed; continuing", exc_info=True) - return 0 + if not run_benchmark: + logger.info("run_benchmark=false, skipping main benchmark") + try: + from projects.rhaiis.orchestration.analysis import run_standalone_analysis + + run_standalone_analysis( + model_cfg, + accelerator_key, + engine_args, + run_uuid=run_uuid, + restrict_profiles=[workload_key], + ) + except Exception: + logger.warning("Standalone analysis failed", exc_info=True) + _warnings.append(f"Standalone analysis failed for {workload_key}") + else: + logger.info("Running benchmark at rates=%s for workload=%s", rates, workload_key) + + benchmark_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") + + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=rates, + max_seconds=max_seconds, + ) + + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=_guidellm_job_name("guidellm-bench", workload_key, deployment_name), + namespace=namespace, + image=benchmark_image, + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), + ) def _create_test_labels( - model_key: str, workload_key: str, accelerator: str, vllm_args: dict + model_key: str, + workload_key: str, + accelerator: str, + engine_args: dict, + *, + hf_model_id: str = "", + version: str = "", + serving_image: str = "", + cluster_tag: str = "", + accelerator_chip: str = "", + run_uuid: str = "", + trtllm_config: dict | None = None, ) -> None: + _, image_tag = runtime_config.split_image_tag(serving_image) if serving_image else ("", "") + parts = [f"{k}: {v}" for k, v in engine_args.items()] + for key, value in (trtllm_config or {}).items(): + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, (dict, list)) else value + ) + parts.append(f"trtllm.{key}: {formatted_value}") + runtime_args = "; ".join(parts) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) labels = { "model_key": model_key, "workload_key": workload_key, - "accelerator": accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), + "accelerator": accelerator_chip or accelerator, + "tensor_parallel_size": str(tp), + "hf_model_id": hf_model_id, + "version": version, + "image_tag": image_tag, + "cluster_tag": cluster_tag, + "runtime_args": runtime_args, + "run_uuid": run_uuid, } write_test_labels(env.ARTIFACT_DIR, labels) logger.info("Created test labels: %s", labels) @@ -179,8 +507,8 @@ def _set_mlflow_metadata( workload_key: str, model_cfg: dict, accelerator: str, - vllm_image: str, - vllm_args: dict, + serving_image: str, + engine_args: dict, benchmark_cfg: dict, rates: list[int], max_seconds: int, @@ -189,18 +517,24 @@ def _set_mlflow_metadata( ) -> None: from projects.core.library import config - image_name, image_tag = runtime_config.split_image_tag(vllm_image) + image_name, image_tag = runtime_config.split_image_tag(serving_image) guidellm_image = benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0") benchmark_args = benchmark_cfg.get("args", {}) + tp = ( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ) tags = { "project": "rhaiis", "model_key": model_key, "hf_model_id": model_cfg["hf_model_id"], "accelerator": accelerator, - "tensor_parallel_size": str(vllm_args.get("tensor-parallel-size", 1)), - "vllm_image": vllm_image, - "vllm_version": image_tag, + "tensor_parallel_size": str(tp), + "serving_image": serving_image, + "serving_version": image_tag, "workload_key": workload_key, "rates": ",".join(str(r) for r in rates), "max_seconds": str(max_seconds), @@ -215,43 +549,307 @@ def _set_mlflow_metadata( logger.info("Set MLflow tags: %s", list(tags.keys())) -def _generate_psap_payload( +def _sync_postprocessed_dashboard_csv(model_key: str, workload_keys: list[str]) -> None: + """Find the dashboard CSV generated by Caliper kpis_to_csv and sync to S3.""" + from pathlib import Path + + from projects.core.library import config + + csv_dashboard_cfg = config.project.get_config("caliper.postprocess.csv_dashboard", {}) + if not csv_dashboard_cfg or not csv_dashboard_cfg.get("enabled", False): + logger.info("Dashboard CSV S3 sync not enabled") + return + + # Find the CSV produced by the kpis_to_csv step in the postprocessing directory + postprocess_dirs = sorted(Path(env.ARTIFACT_DIR).glob("*__postprocessing")) + if not postprocess_dirs: + logger.info("No postprocessing directory found, skipping dashboard CSV sync") + return + + kpis_to_csv_output = config.project.get_config( + "caliper.postprocess.kpi.kpis_to_csv.output", "dashboard.csv" + ) + csv_path = postprocess_dirs[-1] / kpis_to_csv_output + if not csv_path.exists(): + logger.info("Dashboard CSV not found at %s, skipping sync", csv_path) + return + + logger.info("Found postprocessed dashboard CSV: %s", csv_path) + + from projects.rhaiis.postprocess.s3_dashboard import sync_csv_to_s3 + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + sync_result = sync_csv_to_s3( + csv_path, + s3_bucket=s3_cfg.get("bucket", ""), + s3_key=csv_dashboard_cfg.get("s3_key", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Dashboard CSV sync result: %s", sync_result) + + version = config.project.get_config("tests.rhaiis.version", "") + compare_version = config.project.get_config("tests.rhaiis.compare_version", "") + if not compare_version or not version: + return + + model_cfg = runtime_config.get_model(model_key) + accelerator = runtime_config.get_accelerator() + engine = runtime_config.get_engine() + engine_defaults = runtime_config.get_engine_args(engine) + first_workload = runtime_config.get_workload(workload_keys[0]) + ea = runtime_config.merge_engine_args(engine_defaults, model_cfg, first_workload, engine) + + from projects.rhaiis.orchestration.analysis import run_regression_check + + run_regression_check( + csv_path, + compare_version, + version, + model_cfg, + accelerator, + run_uuid="", + engine_args=ea, + ) + + +def _upload_predictor_log(run_uuid: str) -> None: + """Upload the captured predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_predictor_log_to_s3 + + matches = sorted( + Path(env.ARTIFACT_DIR).glob("*__capture_isvc_state/artifacts/inferenceservice.pods.logs") + ) + log_path = matches[-1] if matches else None + if not log_path or not log_path.exists(): + logger.info("No predictor pod log found under %s, skipping upload", env.ARTIFACT_DIR) + return + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + + result = upload_predictor_log_to_s3( + log_path, + run_uuid=run_uuid, + s3_bucket=s3_cfg.get("bucket", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), + dry_run=config.project.get_config("caliper.export.dry_run", False), + ) + logger.info("Predictor log upload result: %s", result) + + +def _run_warmup_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, model_cfg: dict, - accelerator: str, - vllm_image: str, - vllm_args: dict, + workload: dict, workload_key: str, + benchmark_timeout: int, ) -> None: - from pathlib import Path + """Run a short warmup benchmark to prime KV cache and CUDA kernels.""" + from projects.core.library import config + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, + ) - from projects.rhaiis.postprocess.parser import generate_psap_payload, write_psap_payload + warmup_cfg = config.project.get_config("rhaiis.warmup", {}) + warmup_rate = warmup_cfg.get("rate", 200) + warmup_max_seconds = warmup_cfg.get("max_seconds", 60) - matches = list( - Path(env.ARTIFACT_DIR).glob("*__run_guidellm_benchmark/artifacts/results/benchmarks.json") + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=[warmup_rate], + max_seconds=warmup_max_seconds, ) - if not matches: - logger.warning( - "benchmarks.json not found under %s, skipping PSAP payload", env.ARTIFACT_DIR + + logger.info("Running warmup (concurrency=%d, duration=%ds)", warmup_rate, warmup_max_seconds) + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=_guidellm_job_name("guidellm-warmup", workload_key, deployment_name), + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), ) - return - benchmarks_json = matches[0] + logger.info("Warmup completed") + except Exception: + logger.warning("Warmup failed; continuing with benchmark", exc_info=True) - payload = generate_psap_payload( - benchmarks_json_path=benchmarks_json, - model_id=model_cfg["hf_model_id"], - vllm_image=vllm_image, - vllm_args=vllm_args, - accelerator=accelerator, - workload_key=workload_key, + +def _run_profiler_step( + *, + deployment_name: str, + namespace: str, + endpoint_url: str, + benchmark_cfg: dict, + model_cfg: dict, + workload: dict, + workload_key: str, + benchmark_timeout: int, +) -> None: + """Run profiler-gated benchmarks: verify prereqs → enable gate → benchmark → disable gate → copy traces.""" + from projects.guidellm.toolbox.run_guidellm_benchmark.main import ( + run as run_guidellm_benchmark, ) - output_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "results" - write_psap_payload( - payload=payload, - output_dir=output_dir, + from projects.rhaiis.toolbox.copy_profiler_traces.main import run as copy_profiler_traces + from projects.rhaiis.toolbox.enable_profiler_gate.main import run as enable_profiler_gate + from projects.rhaiis.toolbox.verify_profiler_prereqs.main import run as verify_profiler_prereqs + + logger.info("Verifying profiler prerequisites") + verify_profiler_prereqs(namespace=namespace) + + profiler_cfg = runtime_config.get_profiler_config() + labels = profiler_cfg.get("labels", []) + if not labels: + labels = [_derive_profiler_label(workload)] + logger.info("Auto-generated profiler label from workload: %s", labels[0]) + + profiler_max_seconds = profiler_cfg.get("max_seconds", 60) + + for label in labels: + logger.info("Profiling label=%s", label) + + gate_value = label if isinstance(label, str) else str(label) + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + gate_value=gate_value, + ) + + profiler_rates = profiler_cfg.get("rates", [1]) + guidellm_args = runtime_config.build_guidellm_args( + benchmark_cfg=benchmark_cfg, + model_id=model_cfg["hf_model_id"], + data=workload["data"], + rates=profiler_rates, + max_seconds=profiler_max_seconds, + ) + + try: + run_guidellm_benchmark( + endpoint_url=f"{endpoint_url}/v1", + name=_guidellm_job_name("guidellm-profiler", workload_key, deployment_name), + namespace=namespace, + image=benchmark_cfg.get("image", "ghcr.io/vllm-project/guidellm:v0.6.0"), + timeout=benchmark_timeout, + pvc_size=benchmark_cfg.get("pvc_size", "5Gi"), + guidellm_args=guidellm_args, + hf_token_secret=benchmark_cfg.get("hf_token_secret", ""), + fs_group=benchmark_cfg.get("fs_group"), + ) + finally: + enable_profiler_gate( + name=deployment_name, + namespace=namespace, + disable=True, + ) + + logger.info("Copying profiler traces from pod") + try: + copy_profiler_traces(name=deployment_name, namespace=namespace) + except Exception: + logger.warning("Failed to copy profiler traces", exc_info=True) + + +def _derive_profiler_label(workload: dict) -> str: + """Auto-generate a profiler label like 'isl1000_osl1000' from the workload data string.""" + data = workload.get("data", "") + params = dict(item.split("=", 1) for item in data.split(",") if "=" in item) + isl = params.get("prompt_tokens", "0") + osl = params.get("output_tokens", "0") + return f"isl{isl}_osl{osl}" + + +def _infer_profile_labels_from_traces(trace_files: list) -> list[str]: + """Extract unique profile labels from trace filenames. + + Filenames follow the pattern: trace_rank{R}_pid{P}_run{LABEL}_range{S}-{E}.json + The {LABEL} portion is the gate value used during profiling (e.g. 'isl1000_osl1000'). + """ + import re + + pattern = re.compile(r"_run(.+?)_range\d+-\d+") + labels: list[str] = [] + seen: set[str] = set() + for f in trace_files: + m = pattern.search(f.name) + if m and m.group(1) not in seen: + seen.add(m.group(1)) + labels.append(m.group(1)) + return labels + + +def _upload_profiler_traces( + model_cfg: dict, + accelerator: str, + engine_args: dict, + profiler_cfg: dict, +) -> None: + from pathlib import Path + + from projects.core.library import config + from projects.rhaiis.postprocess.s3_dashboard import upload_profiler_traces_to_s3 + + trace_files = sorted( + Path(env.ARTIFACT_DIR).glob("*__copy_profiler_traces/artifacts/traces/trace_*") + ) + if not trace_files: + logger.info("No profiler traces to upload") + return + + traces_dir = trace_files[0].parent + if len({f.parent for f in trace_files}) > 1: + traces_dir = Path(env.ARTIFACT_DIR) / "artifacts" / "traces_combined" + traces_dir.mkdir(parents=True, exist_ok=True) + for f in trace_files: + import shutil + + shutil.copy2(f, traces_dir / f.name) + logger.info("Found %d profiler trace files in %s", len(trace_files), traces_dir) + + version = config.project.get_config("tests.rhaiis.version", "") + if not version: + logger.info("No version configured, skipping profiler trace upload") + return + + profile_labels = profiler_cfg.get("labels", []) + if not profile_labels: + profile_labels = _infer_profile_labels_from_traces(trace_files) + logger.info("Inferred profile labels from trace filenames: %s", profile_labels) + + s3_cfg = config.project.get_config("rhaiis.s3", {}) + result = upload_profiler_traces_to_s3( + traces_dir, + model_name=model_cfg.get("hf_model_id", ""), accelerator=accelerator, - model_id=model_cfg["hf_model_id"], - workload_key=workload_key, + tp_size=int( + engine_args.get("tensor-parallel-size") + or engine_args.get("tp-size") + or engine_args.get("tp_size") + or 1 + ), + version=version, + profile_labels=profile_labels, + s3_bucket=s3_cfg.get("bucket", ""), + s3_prefix=profiler_cfg.get("s3_prefix", ""), + vault_name=s3_cfg.get("vault", ""), + credentials_file=s3_cfg.get("credentials_file", "aws.credentials"), + dry_run=config.project.get_config("caliper.export.dry_run", False), ) + logger.info("Profiler trace upload result: %s", result) def _capture_and_cleanup(deployment_name: str, namespace: str) -> None: diff --git a/projects/rhaiis/orchestration/test_rhaiis.py b/projects/rhaiis/orchestration/test_rhaiis.py index e2a125b89..69be8db73 100644 --- a/projects/rhaiis/orchestration/test_rhaiis.py +++ b/projects/rhaiis/orchestration/test_rhaiis.py @@ -14,8 +14,12 @@ namespace="rhaiis.namespace", ) def test(_cfg): + workload_keys = config.project.get_config("tests.rhaiis.workload_keys", []) + if not workload_keys: + workload_keys = [_cfg.workload_key] + test_phase.run( model_key=_cfg.model_key, - workload_key=_cfg.workload_key, + workload_keys=workload_keys, namespace=_cfg.namespace, ) diff --git a/projects/rhaiis/postprocess/agent.py b/projects/rhaiis/postprocess/agent.py new file mode 100644 index 000000000..51e5591a6 --- /dev/null +++ b/projects/rhaiis/postprocess/agent.py @@ -0,0 +1,303 @@ +"""Client for the PSAP AI agent's /v1/stream endpoint. + +Sends regression context as a natural-language prompt and collects the +streamed AI response into a markdown string. Ported from model-furnace. +""" + +import json +import logging +import uuid +from datetime import UTC +from urllib.error import URLError +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +AGENT_TIMEOUT_SECONDS = 600 +AGENT_HEALTH_TIMEOUT = 15 +AGENT_SEVERITY_THRESHOLD = 10 + + +def check_agent_connectivity(agent_url: str) -> tuple[bool, str]: + """Verify that the agent endpoint is reachable. + + Tries a lightweight GET to the agent health URL. Returns (ok, detail). + """ + url = agent_url + base_url = url.rsplit("/", 2)[0] + health_url = f"{base_url}/health" + + try: + req = Request(health_url, method="GET") + resp = urlopen(req, timeout=AGENT_HEALTH_TIMEOUT) # noqa: S310 + status = resp.getcode() + if status and status < 400: + return True, f"Agent reachable at {health_url} (HTTP {status})" + return False, f"Agent returned HTTP {status} at {health_url}" + except (URLError, OSError) as e: + return False, f"Cannot reach agent at {health_url}: {e}" + except Exception as e: + return False, f"Unexpected error checking agent at {health_url}: {e}" + + +def _build_prompt( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + improvements: list | None = None, +) -> str: + """Build a natural-language prompt for the agent from regression context.""" + lines = [ + f"Analyze the performance regression between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}.", + "", + "The following metrics regressed significantly (>10%):", + ] + for r in severe_regressions: + direction = "dropped" if r["pct_diff"] < 0 else "increased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + if improvements: + lines.append("") + lines.append("The following metrics improved significantly:") + for r in improvements: + direction = "increased" if r["pct_diff"] > 0 else "decreased" + lines.append( + f"- {r['metric']} ({r['profile']}): {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} -> {r['current']:.2f})" + ) + lines.append("") + lines.append( + f"Check if the runtime args differ between {current_version} and {compare_version} " + f"for model {model} on {accelerator} with TP={tp}. " + ) + lines.append("") + lines.append( + "Provide root cause analysis using pytorch profiler traces, vLLM logs, " + "and vLLM source code where available." + ) + return "\n".join(lines) + + +def build_pr_followup_prompt( + current_version: str, + compare_version: str, +) -> str: + """Build a followup prompt asking the agent to identify relevant PRs.""" + return ( + "Based on your previous analysis, use the compare_vllm_versions and " + "get_vllm_pull_request tools to identify which specific pull requests " + f"between vLLM {compare_version} and {current_version} most likely " + "explain the performance differences you found. For each PR you " + "identify, include its GitHub link and a brief explanation of why " + "it's relevant to the regressions or improvements you observed." + ) + + +def request_agent_analysis( + model: str, + accelerator: str, + current_version: str, + compare_version: str, + tp: str, + severe_regressions: list, + agent_url: str, + job_id: str = "", + improvements: list | None = None, +) -> str | None: + """Send regression context to the PSAP agent and return its analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url + + prompt = _build_prompt( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + tp=tp, + severe_regressions=severe_regressions, + improvements=improvements, + ) + + session_key = f"forge-rhaiis-{job_id or uuid.uuid4()}" + body = { + "message": prompt, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Requesting agent analysis for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent analysis received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent analysis for job %s: %s", job_id, e) + return None + + +def send_followup(message: str, job_id: str, agent_url: str) -> str | None: + """Send a followup message to the agent on an existing session. + + Reuses the same thread_id/session_id as request_agent_analysis so the + agent retains full conversation context from the initial analysis. + + Returns the agent's response as a markdown string, or None on failure. + """ + url = agent_url + + session_key = f"forge-rhaiis-{job_id}" + body = { + "message": message, + "thread_id": session_key, + "session_id": session_key, + "user_id": "forge-rhaiis", + "stream_tokens": False, + "model": "claude-opus-4-6", + } + + logger.info("Sending agent followup for job %s", job_id) + + try: + req = Request( + url, + data=json.dumps(body).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + resp = urlopen(req, timeout=AGENT_TIMEOUT_SECONDS) # noqa: S310 + ai_content = _collect_response(resp) + + if ai_content: + logger.info("Agent followup received for job %s (%d chars)", job_id, len(ai_content)) + else: + logger.warning("Agent returned empty followup response for job %s", job_id) + + return ai_content + + except (URLError, OSError) as e: + logger.error("Agent followup request failed for job %s: %s", job_id, e) + return None + except Exception as e: + logger.error("Unexpected error during agent followup for job %s: %s", job_id, e) + return None + + +def _collect_response(resp) -> str | None: + """Read the NDJSON stream and extract the final AI message content.""" + collected_tokens = [] + final_message = None + + for raw_line in resp: + line = raw_line.decode("utf-8", errors="replace").strip() + if not line or line == "[DONE]": + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + event_type = event.get("type") + content = event.get("content") + + if event_type == "token" and isinstance(content, str): + collected_tokens.append(content) + elif ( + event_type == "message" + and isinstance(content, dict) + and content.get("type") == "ai" + and content.get("content") + ): + final_message = content["content"] + + if final_message: + return final_message + if collected_tokens: + return "".join(collected_tokens) + return None + + +def markdown_to_html( + md_text: str, + job_id: str, + model: str, + current_version: str, + compare_version: str, +) -> str: + """Convert markdown analysis to a self-contained HTML page.""" + try: + import markdown as md_lib + + body = md_lib.markdown(md_text, extensions=["tables", "fenced_code"]) + except ImportError: + import html as html_lib + + body = f"
{html_lib.escape(md_text)}
" + + from datetime import datetime + + timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC") + + return f""" + + + + +Analysis — {job_id} + + + +

Regression Analysis Report

+
+ Job: {job_id} + Model: {model} + Versions: {current_version} vs {compare_version} + Generated: {timestamp} +
+{body} + +""" diff --git a/projects/rhaiis/postprocess/csv_export.py b/projects/rhaiis/postprocess/csv_export.py new file mode 100644 index 000000000..5f86396dc --- /dev/null +++ b/projects/rhaiis/postprocess/csv_export.py @@ -0,0 +1,282 @@ +"""Generate consolidated_dashboard.csv from PSAP JSON files. + +Matches the CSV schema from model-furnace's visualization/parse_furnace_results.py. +""" + +from __future__ import annotations + +import csv +import json +import logging +import re +from pathlib import Path + +logger = logging.getLogger(__name__) + +FIELDNAMES = [ + "run", + "accelerator", + "model", + "version", + "prompt toks", + "output toks", + "TP", + "measured concurrency", + "intended concurrency", + "measured rps", + "output_tok/sec", + "total_tok/sec", + "prompt_token_count_mean", + "prompt_token_count_p99", + "output_token_count_mean", + "output_token_count_p99", + "ttft_median", + "ttft_p95", + "ttft_p1", + "ttft_p999", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p999", + "tpot_p1", + "itl_median", + "itl_p95", + "itl_p999", + "itl_p1", + "request_latency_median", + "request_latency_min", + "request_latency_max", + "successful_requests", + "errored_requests", + "uuid", + "ttft_mean", + "ttft_p99", + "itl_mean", + "itl_p99", + "runtime_args", + "guidellm_start_time_ms", + "guidellm_end_time_ms", + "image_tag", + "guidellm_version", + "DP", + "dataset", + "spec_decoding", + "prefix_caching", + "turns", + "prefix_tokens", + "prefix_count", + "request_type", +] + + +def _parse_accelerator_key(acc_key: str) -> tuple[str, str | None]: + """Return (chip_family, cluster_tag) from an accelerator key. + + Convention: keys follow ``_`` (e.g. H200_HERA). + Keys without an underscore have no cluster tag. + """ + if not acc_key: + return "", None + if "_" in acc_key: + chip_family, cluster_tag = acc_key.split("_", 1) + return chip_family, cluster_tag.lower() + return acc_key, None + + +def _format_runtime_args(inference_server_args: dict, trtllm_config: dict | None = None) -> str: + """Format args into semicolon-separated ``key: value`` string matching model-furnace.""" + if not inference_server_args and not trtllm_config: + return "" + parts = [] + for key, value in (inference_server_args or {}).items(): + formatted_key = key.replace("_", "-") + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) + parts.append(f"{formatted_key}: {formatted_value}") + for key, value in (trtllm_config or {}).items(): + formatted_key = f"trtllm.{key.replace('_', '-')}" + formatted_value = ( + json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value + ) + parts.append(f"{formatted_key}: {formatted_value}") + return "; ".join(parts) + + +def find_psap_files(artifact_dir: Path) -> list[Path]: + """Glob for PSAP_perf*.json files in artifact_dir (recursive).""" + return sorted(artifact_dir.rglob("PSAP_perf*.json")) + + +def generate_dashboard_csv( + psap_json_path: Path, version: str, output_path: Path, *, run_uuid: str = "" +) -> Path: + """Read a PSAP JSON file and produce a dashboard CSV.""" + with open(psap_json_path, encoding="utf-8") as f: + payload = json.load(f) + + model = payload.get("model", "") + acc_key = payload.get("accelerator_type", "") + acc, cluster_tag = _parse_accelerator_key(acc_key) + + inference_server_args = payload.get("inference_server_args", {}) + tp = ( + inference_server_args.get("tensor-parallel-size") + or inference_server_args.get("tp-size") + or inference_server_args.get("tp_size") + or payload.get("accelerator_count", 1) + ) + + image_tag = payload.get("container_image_tag", "") + runtime_args = _format_runtime_args(inference_server_args, payload.get("trtllm_config")) + + report = payload.get("report", {}) + guidellm_version = report.get("metadata", {}).get("guidellm_version", "") + benchmarks = report.get("benchmarks", []) + + data_str = "" + args = report.get("args", {}) + if isinstance(args, dict): + data_list = args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + prompt_toks = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + output_toks = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + start_times = [] + end_times = [] + for bench in benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + guidellm_start_ms = ( + int(min(start_times) * 1000) if start_times else payload.get("guidellm_start_time_ms") + ) + guidellm_end_ms = ( + int(max(end_times) * 1000) if end_times else payload.get("guidellm_end_time_ms") + ) + + rows = [] + for bench in benchmarks: + metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + model_name = f"{acc}-{cluster_tag}-{model}-{tp}" if cluster_tag else f"{acc}-{model}-{tp}" + + row = _extract_row( + metrics=metrics, + strategy=strategy, + model_name=model_name, + model=model, + accelerator=acc, + version=version, + tp=tp, + prompt_toks=prompt_toks, + output_toks=output_toks, + image_tag=image_tag, + runtime_args=runtime_args, + guidellm_start_ms=guidellm_start_ms, + guidellm_end_ms=guidellm_end_ms, + guidellm_version=guidellm_version, + run_uuid=run_uuid, + ) + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES) + writer.writeheader() + writer.writerows(rows) + + logger.info("Generated dashboard CSV with %d rows: %s", len(rows), output_path) + return output_path + + +def _extract_row( + *, + metrics: dict, + strategy: dict, + model_name: str, + model: str, + accelerator: str, + version: str, + tp, + prompt_toks, + output_toks, + image_tag: str, + runtime_args: str, + guidellm_start_ms, + guidellm_end_ms, + guidellm_version: str, + run_uuid: str = "", +) -> dict: + def _pct(metric_name: str, pct: str): + return metrics.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + + def _stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("successful", {}).get(stat) + + def _total_stat(metric_name: str, stat: str): + return metrics.get(metric_name, {}).get("total", {}).get(stat) + + request_totals = metrics.get("request_totals", {}) + successful = request_totals.get("successful", 0) + errored = request_totals.get("errored", 0) + + return { + "run": model_name, + "accelerator": accelerator, + "model": model, + "version": version, + "prompt toks": prompt_toks, + "output toks": output_toks, + "TP": tp, + "measured concurrency": _stat("request_concurrency", "mean"), + "intended concurrency": strategy.get("streams"), + "measured rps": _stat("requests_per_second", "mean"), + "output_tok/sec": _total_stat("output_tokens_per_second", "mean"), + "total_tok/sec": _total_stat("tokens_per_second", "mean"), + "prompt_token_count_mean": _stat("prompt_token_count", "mean"), + "prompt_token_count_p99": _pct("prompt_token_count", "p99"), + "output_token_count_mean": _stat("output_token_count", "mean"), + "output_token_count_p99": _pct("output_token_count", "p99"), + "ttft_median": _stat("time_to_first_token_ms", "median"), + "ttft_p95": _pct("time_to_first_token_ms", "p95"), + "ttft_p1": _pct("time_to_first_token_ms", "p01"), + "ttft_p999": _pct("time_to_first_token_ms", "p999"), + "tpot_median": _stat("time_per_output_token_ms", "median"), + "tpot_p95": _pct("time_per_output_token_ms", "p95"), + "tpot_p99": _pct("time_per_output_token_ms", "p99"), + "tpot_p999": _pct("time_per_output_token_ms", "p999"), + "tpot_p1": _pct("time_per_output_token_ms", "p01"), + "itl_median": _stat("inter_token_latency_ms", "median"), + "itl_p95": _pct("inter_token_latency_ms", "p95"), + "itl_p999": _pct("inter_token_latency_ms", "p999"), + "itl_p1": _pct("inter_token_latency_ms", "p01"), + "request_latency_median": _stat("request_latency", "median"), + "request_latency_min": _stat("request_latency", "min"), + "request_latency_max": _stat("request_latency", "max"), + "successful_requests": successful, + "errored_requests": errored, + "uuid": run_uuid, + "ttft_mean": _stat("time_to_first_token_ms", "mean"), + "ttft_p99": _pct("time_to_first_token_ms", "p99"), + "itl_mean": _stat("inter_token_latency_ms", "mean"), + "itl_p99": _pct("inter_token_latency_ms", "p99"), + "runtime_args": runtime_args, + "guidellm_start_time_ms": guidellm_start_ms or "", + "guidellm_end_time_ms": guidellm_end_ms or "", + "image_tag": image_tag, + "guidellm_version": guidellm_version, + "DP": "", + "dataset": "", + "spec_decoding": "", + "prefix_caching": "", + "turns": "", + "prefix_tokens": "", + "prefix_count": "", + "request_type": "", + } diff --git a/projects/rhaiis/postprocess/kpis.py b/projects/rhaiis/postprocess/kpis.py index 7704bd50c..fc137feea 100644 --- a/projects/rhaiis/postprocess/kpis.py +++ b/projects/rhaiis/postprocess/kpis.py @@ -5,143 +5,57 @@ from projects.caliper.engine.model import UnifiedRunModel +# All per-rate-point metrics we emit as KPIs. +# (kpi_id, performance_curves key, unit, higher_is_better) +_CURVE_KPI_MAPPINGS: list[tuple[str, str, str, bool | None]] = [ + # Throughput + ("rhaiis_output_tok_per_sec", "output_tok_per_sec", "tokens/s", True), + ("rhaiis_total_tok_per_sec", "total_tok_per_sec", "tokens/s", True), + # Concurrency / rate + ("rhaiis_measured_concurrency", "request_concurrency", "count", None), + ("rhaiis_measured_rps", "measured_rps", "req/s", True), + ("rhaiis_intended_concurrency", "intended_concurrency", "count", None), + # Request counts + ("rhaiis_completed_requests", "successful_requests", "count", True), + ("rhaiis_failed_requests", "errored_requests", "count", False), + # TTFT + ("rhaiis_ttft_median", "ttft_median", "s", False), + ("rhaiis_ttft_p95", "ttft_p95", "s", False), + ("rhaiis_ttft_p99", "ttft_p99", "s", False), + ("rhaiis_ttft_p1", "ttft_p1", "s", False), + ("rhaiis_ttft_p999", "ttft_p999", "s", False), + ("rhaiis_ttft_mean", "ttft_mean", "s", False), + # TPOT + ("rhaiis_tpot_median", "tpot_median", "s", False), + ("rhaiis_tpot_p95", "tpot_p95", "s", False), + ("rhaiis_tpot_p99", "tpot_p99", "s", False), + ("rhaiis_tpot_p1", "tpot_p1", "s", False), + ("rhaiis_tpot_p999", "tpot_p999", "s", False), + # ITL + ("rhaiis_itl_median", "itl_median", "s", False), + ("rhaiis_itl_p95", "itl_p95", "s", False), + ("rhaiis_itl_p99", "itl_p99", "s", False), + ("rhaiis_itl_p1", "itl_p1", "s", False), + ("rhaiis_itl_p999", "itl_p999", "s", False), + ("rhaiis_itl_mean", "itl_mean", "s", False), + # Request latency + ("rhaiis_request_latency_median", "request_latency_median", "s", False), + ("rhaiis_request_latency_min", "request_latency_min", "s", False), + ("rhaiis_request_latency_max", "request_latency_max", "s", False), + # Token counts + ("rhaiis_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), + ("rhaiis_prompt_token_count_p99", "prompt_token_count_p99", "tokens", None), + ("rhaiis_output_token_count_mean", "output_token_count_mean", "tokens", None), + ("rhaiis_output_token_count_p99", "output_token_count_p99", "tokens", None), +] + class RhaiisKpiHandler: @staticmethod def get_catalog() -> list[dict[str, Any]]: return [ - { - "kpi_id": "guidellm_request_rate", - "name": "Request Rate", - "unit": "req/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_tokens_per_second", - "name": "Total Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_input_tokens_per_second", - "name": "Input Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_output_tokens_per_second", - "name": "Output Token Throughput", - "unit": "tokens/s", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_ttft_median", - "name": "Time to First Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p95", - "name": "Time to First Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_ttft_p99", - "name": "Time to First Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_median", - "name": "Inter Token Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p95", - "name": "Inter Token Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_itl_p99", - "name": "Inter Token Latency (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_median", - "name": "Time Per Output Token (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p95", - "name": "Time Per Output Token (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_tpot_p99", - "name": "Time Per Output Token (P99)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_median", - "name": "End-to-End Request Latency (Median)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_latency_p95", - "name": "End-to-End Request Latency (P95)", - "unit": "s", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_input_tokens_per_request", - "name": "Input Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_output_tokens_per_request", - "name": "Output Tokens per Request", - "unit": "tokens", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_request_concurrency", - "name": "Measured Request Concurrency", - "unit": "count", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_completed_requests", - "name": "Completed Requests", - "unit": "count", - "higher_is_better": True, - }, - { - "kpi_id": "guidellm_failed_requests", - "name": "Failed Requests", - "unit": "count", - "higher_is_better": False, - }, - { - "kpi_id": "guidellm_duration", - "name": "Benchmark Duration", - "unit": "s", - "higher_is_better": None, - }, - { - "kpi_id": "guidellm_prompt_token_count_mean", - "name": "Prompt Token Count (Mean)", - "unit": "tokens", - "higher_is_better": None, - }, + {"kpi_id": kpi_id, "name": kpi_id, "unit": unit, "higher_is_better": hib} + for kpi_id, _, unit, hib in _CURVE_KPI_MAPPINGS ] @staticmethod @@ -156,66 +70,61 @@ def compute_kpis(model: UnifiedRunModel) -> list[dict[str, Any]]: continue base_labels = {**r.distinguishing_labels} + curves = r.metrics.get("performance_curves", {}) + request_rates = r.metrics.get("request_rate", []) + + # Report-level metadata — same for every rate point in this record + meta_labels = { + "guidellm_version": str(r.metrics.get("guidellm_version", "")), + "prompt_toks": str(r.metrics.get("prompt_toks", "")), + "output_toks": str(r.metrics.get("output_toks", "")), + "guidellm_start_time_ms": str(r.metrics.get("guidellm_start_time_ms", "")), + "guidellm_end_time_ms": str(r.metrics.get("guidellm_end_time_ms", "")), + } + + if not curves or not request_rates: + continue + + n_rates = len(request_rates) + + for idx in range(n_rates): + rate_labels = { + **base_labels, + **meta_labels, + "rate_index": str(idx), + } + + for kpi_id, curve_key, unit, higher_is_better in _CURVE_KPI_MAPPINGS: + curve_values = curves.get(curve_key, []) + if idx >= len(curve_values): + continue + raw_value = curve_values[idx] + if raw_value is None: + continue + try: + value = float(raw_value) + except (TypeError, ValueError): + continue + + labels = {**rate_labels} + if higher_is_better is not None: + labels["higher_is_better"] = higher_is_better - kpi_mappings = [ - ("guidellm_request_rate", "request_rate", "req/s", True), - ("guidellm_tokens_per_second", "tokens_per_second", "tokens/s", True), - ("guidellm_input_tokens_per_second", "input_tokens_per_second", "tokens/s", True), - ("guidellm_output_tokens_per_second", "output_tokens_per_second", "tokens/s", True), - ("guidellm_ttft_median", "ttft_median", "s", False), - ("guidellm_ttft_p95", "ttft_p95", "s", False), - ("guidellm_ttft_p99", "ttft_p99", "s", False), - ("guidellm_itl_median", "itl_median", "s", False), - ("guidellm_itl_p95", "itl_p95", "s", False), - ("guidellm_itl_p99", "itl_p99", "s", False), - ("guidellm_tpot_median", "tpot_median", "s", False), - ("guidellm_tpot_p95", "tpot_p95", "s", False), - ("guidellm_tpot_p99", "tpot_p99", "s", False), - ("guidellm_request_latency_median", "request_latency_median", "s", False), - ("guidellm_request_latency_p95", "request_latency_p95", "s", False), - ("guidellm_input_tokens_per_request", "input_tokens_per_request", "tokens", False), - ( - "guidellm_output_tokens_per_request", - "output_tokens_per_request", - "tokens", - False, - ), - ("guidellm_request_concurrency", "request_concurrency", "count", None), - ("guidellm_completed_requests", "completed_requests", "count", True), - ("guidellm_failed_requests", "failed_requests", "count", False), - ("guidellm_duration", "duration", "s", None), - ("guidellm_prompt_token_count_mean", "prompt_token_count_mean", "tokens", None), - ] - - for kpi_id, metric_key, unit, higher_is_better in kpi_mappings: - raw_value = r.metrics.get(metric_key) - try: - value = float(raw_value) if raw_value is not None else None - except (TypeError, ValueError): - value = None - - # Skip KPIs with null values - if value is None: - continue - - labels = {**base_labels} - if higher_is_better is not None: - labels["higher_is_better"] = higher_is_better - - out.append( - { - "schema_version": "1", - "kpi_id": kpi_id, - "value": value, - "unit": unit, - "run_path": r.test_base_path, - "timestamp": ts, - "labels": labels, - "source": { - "test_base_path": r.test_base_path, - "plugin_module": model.plugin_module, - }, - } - ) + out.append( + { + "schema_version": "1", + "kpi_id": kpi_id, + "value": value, + "unit": unit, + "run_id": r.test_base_path, + "run_path": r.test_base_path, + "timestamp": ts, + "labels": labels, + "source": { + "test_base_path": r.test_base_path, + "plugin_module": model.plugin_module, + }, + } + ) return out diff --git a/projects/rhaiis/postprocess/parser.py b/projects/rhaiis/postprocess/parser.py index 31059e2ea..a607811f8 100644 --- a/projects/rhaiis/postprocess/parser.py +++ b/projects/rhaiis/postprocess/parser.py @@ -2,9 +2,7 @@ import json import logging -import uuid -from datetime import UTC, datetime -from pathlib import Path +import re from typing import Any from projects.caliper.engine.model import ( @@ -36,8 +34,12 @@ def parse(self, nodes: list[TestBaseNode]) -> ParseResult: node = _find_node_for_record(record, nodes) if node: - extra = _extract_extra_metrics(node) - merged_metrics = {**record.metrics, **extra} + extra_metrics, extra_curves = _extract_extra_metrics(node) + merged_metrics = {**record.metrics, **extra_metrics} + if extra_curves: + existing_curves = merged_metrics.get("performance_curves", {}) + existing_curves.update(extra_curves) + merged_metrics["performance_curves"] = existing_curves enriched_records.append( UnifiedResultRecord( test_base_path=record.test_base_path, @@ -63,119 +65,163 @@ def _find_node_for_record( return None -def _extract_extra_metrics(node: TestBaseNode) -> dict[str, Any]: - extra: dict[str, Any] = {} - benchmarks_files = [p for p in node.artifact_paths if p.name == "benchmarks.json"] - if not benchmarks_files: - return extra - - try: - data = json.loads(benchmarks_files[0].read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return extra - - benchmarks = data.get("benchmarks", []) - if not benchmarks: - return extra +def _bench_sort_key(bench: dict) -> float: + """Sort key matching GuideLLMParser: requests_per_second.successful.mean.""" + return float( + bench.get("metrics", {}).get("requests_per_second", {}).get("successful", {}).get("mean", 0) + ) - bench = benchmarks[0] - metrics = bench.get("metrics", {}) - def _percentile(metric_name: str, pct: str, default: float = 0.0) -> float: +def _extract_extra_metrics(node: TestBaseNode) -> tuple[dict[str, Any], dict[str, list]]: + """Return (scalar_metrics, extra_curves) extracted from raw benchmarks.json files.""" + extra: dict[str, Any] = {} + benchmarks_files = sorted( + [ + p + for p in node.artifact_paths + if p.name == "benchmarks.json" + or (p.name.startswith("benchmarks-rate-") and p.suffix == ".json") + ], + key=lambda p: p.name, + ) + if not benchmarks_files: + return extra, {} + + all_benchmarks: list[dict] = [] + report_metadata: dict[str, Any] = {} + report_args: dict[str, Any] = {} + + for bf in benchmarks_files: + try: + data = json.loads(bf.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + continue + all_benchmarks.extend(data.get("benchmarks", [])) + if not report_metadata: + report_metadata = data.get("metadata", {}) + if not report_args: + report_args = data.get("args", {}) + + if not all_benchmarks: + return extra, {} + + # Sort by request_rate to match GuideLLMParser curve ordering + all_benchmarks.sort(key=_bench_sort_key) + + # Report-level metadata + extra["guidellm_version"] = report_metadata.get("guidellm_version", "") + + data_str = "" + if isinstance(report_args, dict): + data_list = report_args.get("data", []) + if data_list: + data_str = data_list[0] if isinstance(data_list, list) else str(data_list) + tokens = dict(re.findall(r"(\w+)=([\d.]+)", data_str)) + extra["prompt_toks"] = int(float(tokens["prompt_tokens"])) if "prompt_tokens" in tokens else "" + extra["output_toks"] = int(float(tokens["output_tokens"])) if "output_tokens" in tokens else "" + + start_times = [] + end_times = [] + for bench in all_benchmarks: + sched = bench.get("scheduler_metrics", {}) + if "start_time" in sched: + start_times.append(sched["start_time"]) + if "end_time" in sched: + end_times.append(sched["end_time"]) + extra["guidellm_start_time_ms"] = int(min(start_times) * 1000) if start_times else "" + extra["guidellm_end_time_ms"] = int(max(end_times) * 1000) if end_times else "" + + # Scalar extras from first benchmark (backward compat) + bench0 = all_benchmarks[0] + m0 = bench0.get("metrics", {}) + + def _percentile0(metric_name: str, pct: str, default: float = 0.0) -> float: return float( - metrics.get(metric_name, {}) - .get("successful", {}) - .get("percentiles", {}) - .get(pct, default) + m0.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct, default) ) - def _stat(metric_name: str, stat: str, default: float = 0.0) -> float: - return float(metrics.get(metric_name, {}).get("successful", {}).get(stat, default)) + def _stat0(metric_name: str, stat: str, default: float = 0.0) -> float: + return float(m0.get(metric_name, {}).get("successful", {}).get(stat, default)) - extra["ttft_p99"] = _percentile("time_to_first_token_ms", "p99") / 1000.0 - extra["tpot_p99"] = _percentile("time_per_output_token_ms", "p99") / 1000.0 - extra["itl_p99"] = _percentile("inter_token_latency_ms", "p99") / 1000.0 + extra["ttft_p99"] = _percentile0("time_to_first_token_ms", "p99") / 1000.0 + extra["tpot_p99"] = _percentile0("time_per_output_token_ms", "p99") / 1000.0 + extra["itl_p99"] = _percentile0("inter_token_latency_ms", "p99") / 1000.0 - request_totals = metrics.get("request_totals", {}) - extra["completed_requests"] = int(request_totals.get("successful", 0)) - extra["failed_requests"] = int(request_totals.get("errored", 0)) + request_totals0 = m0.get("request_totals", {}) + extra["completed_requests"] = int(request_totals0.get("successful", 0)) + extra["failed_requests"] = int(request_totals0.get("errored", 0)) + extra["prompt_token_count_mean"] = _stat0("prompt_token_count", "mean") - extra["prompt_token_count_mean"] = _stat("prompt_token_count", "mean") - - concurrency = _stat("request_concurrency", "mean") + concurrency = _stat0("request_concurrency", "mean") if concurrency > 0: extra["request_concurrency"] = concurrency - return extra - - -def generate_psap_payload( - *, - benchmarks_json_path: Path, - model_id: str, - vllm_image: str, - vllm_args: dict[str, Any], - accelerator: str, - workload_key: str, -) -> dict[str, Any]: - report = json.loads(benchmarks_json_path.read_text(encoding="utf-8")) - - image, tag = _split_image_tag(vllm_image) - tp_size = int(vllm_args.get("tensor-parallel-size", 1)) - - guidellm_start = None - guidellm_end = None - benchmarks = report.get("benchmarks", []) - if benchmarks: - starts = [b.get("start_time", 0) for b in benchmarks if b.get("start_time")] - ends = [b.get("end_time", 0) for b in benchmarks if b.get("end_time")] - if starts: - guidellm_start = int(min(starts) * 1000) - if ends: - guidellm_end = int(max(ends) * 1000) - - return { - "experiment_id": str(uuid.uuid4()).upper(), - "experiment_type": "perf", - "model": model_id, - "inference_server": "vllm", - "inference_server_version": tag, - "container_image": image, - "container_image_tag": tag, - "container_entrypoint": None, - "inference_server_args": dict(vllm_args), - "accelerator_type": accelerator.upper(), - "accelerator_count": tp_size, - "accelerator_memory_gb": 0, - "machine_type": None, - "provider": "redhat", - "report": report, - "timestamp": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S%z"), - "guidellm_start_time_ms": guidellm_start, - "guidellm_end_time_ms": guidellm_end, + # Per-benchmark extra curves — indices align with GuideLLM's sorted curves + extra_curves: dict[str, list] = { + "ttft_p1": [], + "ttft_p999": [], + "ttft_mean": [], + "tpot_p1": [], + "tpot_p999": [], + "itl_p1": [], + "itl_p999": [], + "itl_mean": [], + "request_latency_min": [], + "request_latency_max": [], + "measured_rps": [], + "prompt_token_count_mean": [], + "prompt_token_count_p99": [], + "output_token_count_mean": [], + "output_token_count_p99": [], + "output_tok_per_sec": [], + "total_tok_per_sec": [], + "successful_requests": [], + "errored_requests": [], + "intended_concurrency": [], } - -def write_psap_payload( - *, - payload: dict[str, Any], - output_dir: Path, - accelerator: str, - model_id: str, - workload_key: str, -) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - model_short = model_id.rsplit("/", 1)[-1].lower().replace(".", "-") - date_str = datetime.now(UTC).strftime("%Y%m%d") - filename = f"PSAP_perf_{workload_key}_{accelerator}_{model_short}_{date_str}.json" - path = output_dir / filename - path.write_text(json.dumps(payload, indent=2), encoding="utf-8") - logger.info("PSAP payload written to %s", path) - return path - - -def _split_image_tag(full_image: str) -> tuple[str, str]: - if ":" in full_image: - parts = full_image.rsplit(":", 1) - return parts[0], parts[1] - return full_image, "latest" + for bench in all_benchmarks: + metrics = bench.get("metrics", {}) + strategy = bench.get("config", {}).get("strategy", {}) + + def _pct(metric_name: str, pct: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get("percentiles", {}).get(pct) + + def _pct_s(metric_name: str, pct: str): + v = _pct(metric_name, pct) + return v / 1000.0 if v is not None else None + + def _stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("successful", {}).get(stat) + + def _stat_s(metric_name: str, stat: str): + v = _stat(metric_name, stat) + return v / 1000.0 if v is not None else None + + def _total_stat(metric_name: str, stat: str, _m: dict = metrics): + return _m.get(metric_name, {}).get("total", {}).get(stat) + + request_totals = metrics.get("request_totals", {}) + + extra_curves["ttft_p1"].append(_pct_s("time_to_first_token_ms", "p01")) + extra_curves["ttft_p999"].append(_pct_s("time_to_first_token_ms", "p999")) + extra_curves["ttft_mean"].append(_stat_s("time_to_first_token_ms", "mean")) + extra_curves["tpot_p1"].append(_pct_s("time_per_output_token_ms", "p01")) + extra_curves["tpot_p999"].append(_pct_s("time_per_output_token_ms", "p999")) + extra_curves["itl_p1"].append(_pct_s("inter_token_latency_ms", "p01")) + extra_curves["itl_p999"].append(_pct_s("inter_token_latency_ms", "p999")) + extra_curves["itl_mean"].append(_stat_s("inter_token_latency_ms", "mean")) + extra_curves["request_latency_min"].append(_stat("request_latency", "min")) + extra_curves["request_latency_max"].append(_stat("request_latency", "max")) + extra_curves["measured_rps"].append(_stat("requests_per_second", "mean")) + extra_curves["prompt_token_count_mean"].append(_stat("prompt_token_count", "mean")) + extra_curves["prompt_token_count_p99"].append(_pct("prompt_token_count", "p99")) + extra_curves["output_token_count_mean"].append(_stat("output_token_count", "mean")) + extra_curves["output_token_count_p99"].append(_pct("output_token_count", "p99")) + extra_curves["output_tok_per_sec"].append(_total_stat("output_tokens_per_second", "mean")) + extra_curves["total_tok_per_sec"].append(_total_stat("tokens_per_second", "mean")) + extra_curves["successful_requests"].append(request_totals.get("successful", 0)) + extra_curves["errored_requests"].append(request_totals.get("errored", 0)) + extra_curves["intended_concurrency"].append(strategy.get("streams")) + + return extra, extra_curves diff --git a/projects/rhaiis/postprocess/plugin.py b/projects/rhaiis/postprocess/plugin.py index 3b2809d91..3c6765053 100644 --- a/projects/rhaiis/postprocess/plugin.py +++ b/projects/rhaiis/postprocess/plugin.py @@ -1,6 +1,8 @@ from __future__ import annotations +import csv import logging +from collections import defaultdict from pathlib import Path from typing import Any @@ -16,6 +18,65 @@ logger = logging.getLogger(__name__) +# CSV columns whose KPI values are in seconds but the dashboard expects milliseconds. +# GuideLLM parser converts `*_ms` metrics to seconds; the old CSV pipeline kept them in ms. +_SECONDS_TO_MS_COLUMNS = frozenset( + { + "ttft_median", + "ttft_p95", + "ttft_p99", + "ttft_p1", + "ttft_p999", + "ttft_mean", + "tpot_median", + "tpot_p95", + "tpot_p99", + "tpot_p1", + "tpot_p999", + "itl_median", + "itl_p95", + "itl_p99", + "itl_p1", + "itl_p999", + "itl_mean", + } +) + +# KPI ID → dashboard CSV column name mapping (complete) +_KPI_TO_CSV_COLUMN = { + "rhaiis_output_tok_per_sec": "output_tok/sec", + "rhaiis_total_tok_per_sec": "total_tok/sec", + "rhaiis_measured_concurrency": "measured concurrency", + "rhaiis_measured_rps": "measured rps", + "rhaiis_intended_concurrency": "intended concurrency", + "rhaiis_completed_requests": "successful_requests", + "rhaiis_failed_requests": "errored_requests", + "rhaiis_ttft_median": "ttft_median", + "rhaiis_ttft_p95": "ttft_p95", + "rhaiis_ttft_p99": "ttft_p99", + "rhaiis_ttft_p1": "ttft_p1", + "rhaiis_ttft_p999": "ttft_p999", + "rhaiis_ttft_mean": "ttft_mean", + "rhaiis_tpot_median": "tpot_median", + "rhaiis_tpot_p95": "tpot_p95", + "rhaiis_tpot_p99": "tpot_p99", + "rhaiis_tpot_p1": "tpot_p1", + "rhaiis_tpot_p999": "tpot_p999", + "rhaiis_itl_median": "itl_median", + "rhaiis_itl_p95": "itl_p95", + "rhaiis_itl_p99": "itl_p99", + "rhaiis_itl_p1": "itl_p1", + "rhaiis_itl_p999": "itl_p999", + "rhaiis_itl_mean": "itl_mean", + "rhaiis_request_latency_median": "request_latency_median", + "rhaiis_request_latency_min": "request_latency_min", + "rhaiis_request_latency_max": "request_latency_max", + "rhaiis_prompt_token_count_mean": "prompt_token_count_mean", + "rhaiis_prompt_token_count_p99": "prompt_token_count_p99", + "rhaiis_output_token_count_mean": "output_token_count_mean", + "rhaiis_output_token_count_p99": "output_token_count_p99", +} + class RhaiisPlugin(PostProcessingPlugin): def __init__(self) -> None: @@ -53,6 +114,94 @@ def kpi_catalog(self) -> list[dict[str, Any]]: def compute_kpis(self, model: UnifiedRunModel) -> list[dict[str, Any]]: return self.kpi_handler.compute_kpis(model) + def export_kpis_to_csv( + self, + kpi_records: list[dict[str, Any]], + output_path: Path, + include_header_comments: bool = True, + ) -> str: + """Export KPI records to dashboard-format CSV. + + Groups per-rate-point KPIs by (run_path, rate_index) and pivots them + into rows matching the RHAIIS dashboard CSV schema with all fields + populated. + """ + from projects.rhaiis.postprocess.csv_export import FIELDNAMES + + # Group KPIs by (run_path, rate_index) + groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(dict) + group_labels: dict[tuple[str, str], dict[str, Any]] = {} + + for kpi in kpi_records: + labels = kpi.get("labels", {}) + run_path = kpi.get("run_path", "") + rate_index = labels.get("rate_index", "0") + key = (run_path, rate_index) + + csv_col = _KPI_TO_CSV_COLUMN.get(kpi.get("kpi_id", "")) + if csv_col: + groups[key][csv_col] = kpi.get("value") + + # Capture the richest set of labels for this group + if key not in group_labels or len(labels) > len(group_labels[key]): + group_labels[key] = labels + + # Build CSV rows + rows = [] + for key in sorted(groups): + metrics = groups[key] + labels = group_labels.get(key, {}) + + acc = labels.get("accelerator", "").upper() + cluster_tag = labels.get("cluster_tag", "") + model_id = labels.get("hf_model_id", "") + tp = labels.get("tensor_parallel_size", "1") + version = labels.get("version", "") + + run_name = ( + f"{acc}-{cluster_tag}-{model_id}-{tp}" if cluster_tag else f"{acc}-{model_id}-{tp}" + ) + + row: dict[str, Any] = dict.fromkeys(FIELDNAMES, "") + # Populate all metric columns from KPI values + row.update(metrics) + # Convert latency columns from seconds back to ms for dashboard compat + for col in _SECONDS_TO_MS_COLUMNS: + val = row.get(col) + if val not in ("", None): + try: + row[col] = float(val) * 1000.0 + except (TypeError, ValueError): + pass + # Populate metadata columns from labels + row["run"] = run_name + row["accelerator"] = acc + row["model"] = model_id + row["version"] = version + row["TP"] = tp + row["prompt toks"] = labels.get("prompt_toks", "") + row["output toks"] = labels.get("output_toks", "") + # intended concurrency is populated from the rhaiis_intended_concurrency KPI; + # only fall back to labels if the KPI didn't set it + if not row.get("intended concurrency"): + row["intended concurrency"] = labels.get("intended_concurrency", "") + row["image_tag"] = labels.get("image_tag", "") + row["runtime_args"] = labels.get("runtime_args", "") + row["uuid"] = labels.get("run_uuid", "") + row["guidellm_start_time_ms"] = labels.get("guidellm_start_time_ms", "") + row["guidellm_end_time_ms"] = labels.get("guidellm_end_time_ms", "") + row["guidellm_version"] = labels.get("guidellm_version", "") + rows.append(row) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=FIELDNAMES, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + logger.info("Exported %d dashboard CSV rows to %s", len(rows), output_path) + return str(output_path) + def build_ai_data_payload(self, model: UnifiedRunModel) -> dict[str, Any]: return {} diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py new file mode 100644 index 000000000..b71bec30d --- /dev/null +++ b/projects/rhaiis/postprocess/regression.py @@ -0,0 +1,595 @@ +"""RHAIIS-specific regression analysis and Slack notifications. + +Temporary home for CSV-based regression comparison ported from model-furnace. +Will be replaced by generic Caliper KPI-based analysis once the framework lands. +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +import projects.core.notifications.slack.api as slack_api +from projects.core.library import vault + +logger = logging.getLogger(__name__) + + +RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" + + +def _send_via_topsail_bot( + message: str, *, notification_vault: str | None = None, channel_id: str | None = None +) -> bool: + """Send a Slack message using the topsail bot token from the forge notifications vault.""" + vault_name = notification_vault or "psap-forge-notifications" + try: + token_path = vault.get_vault_content_path(vault_name, "topsail-bot.slack-token") + except Exception: + logger.warning("Cannot resolve topsail bot token from vault %s", vault_name) + return False + + if not token_path or not token_path.exists(): + logger.warning("topsail-bot.slack-token not found in vault %s", vault_name) + return False + + token = token_path.read_text().strip() + client = slack_api.init_client(token) + if not client: + logger.error("Failed to init Slack client with topsail bot token") + return False + + _, ok = slack_api.send_message(client, message=message, channel_id=channel_id) + return ok + + +# --------------------------------------------------------------------------- +# CSV-based regression comparison (ported from model-furnace, temporary) +# --------------------------------------------------------------------------- + + +@dataclass +class RegressionResult: + profile: str + metric_column: str + metric_label: str + current_value: float + baseline_value: float + pct_diff: float + is_regression: bool + is_improvement: bool = False + + +def geometric_mean(values) -> float | None: + positive = ( + values[values > 0].values if hasattr(values, "values") else [v for v in values if v > 0] + ) + if len(positive) == 0: + return None + return float(np.exp(np.mean(np.log(positive)))) + + +def compare_runs( + current_csv_path: str, + consolidated_csv_path: str, + compare_version: str, + current_version: str, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> tuple[list[RegressionResult], str]: + """Compare current run metrics against a baseline version. + + Args: + profile_map: Mapping of (input_tokens, output_tokens) -> profile name. + metrics: Mapping of CSV column -> {label, higher_is_better, threshold}. + + Returns (results, skip_reason). If skip_reason is non-empty, comparison was skipped. + """ + import pandas as pd + + current_df = pd.read_csv(current_csv_path, on_bad_lines="warn") + baseline_df = pd.read_csv(consolidated_csv_path, on_bad_lines="warn") + + for col in current_df.columns: + if current_df[col].dtype == object: + current_df[col] = current_df[col].str.strip() + for col in baseline_df.columns: + if baseline_df[col].dtype == object: + baseline_df[col] = baseline_df[col].str.strip() + + if current_df.empty: + return [], "Current run produced no benchmark data" + + model = current_df["model"].iloc[0] + accelerator = current_df["accelerator"].iloc[0] + tp = current_df["TP"].iloc[0] + + logger.info( + "Comparing %s vs %s for model=%s, accelerator=%s, TP=%s", + current_version, + compare_version, + model, + accelerator, + tp, + ) + + def _assign_profile(row): + try: + isl = int(float(row["prompt toks"])) + osl = int(float(row["output toks"])) + except (ValueError, TypeError): + return None + return profile_map.get((isl, osl)) + + current_df["_profile"] = current_df.apply(_assign_profile, axis=1) + baseline_df["_profile"] = baseline_df.apply(_assign_profile, axis=1) + + baseline_filtered = baseline_df[ + (baseline_df["version"] == compare_version) + & (baseline_df["model"] == model) + & (baseline_df["accelerator"] == accelerator) + & (pd.to_numeric(baseline_df["TP"], errors="coerce") == pd.to_numeric(tp, errors="coerce")) + ] + + if baseline_filtered.empty: + return ( + [], + f"No baseline data for {compare_version} with model={model}, accelerator={accelerator}, TP={tp}", + ) + + current_profiles = set(current_df["_profile"].dropna().unique()) + baseline_profiles = set(baseline_filtered["_profile"].dropna().unique()) + common_profiles = sorted(current_profiles & baseline_profiles) + + if restrict_profiles: + common_profiles = [p for p in common_profiles if p in restrict_profiles] + + if not common_profiles: + return [], "No common ISL/OSL profiles between current and baseline" + + results: list[RegressionResult] = [] + + for profile in common_profiles: + cur = current_df[current_df["_profile"] == profile].copy() + base = baseline_filtered[baseline_filtered["_profile"] == profile].copy() + + cur["intended concurrency"] = pd.to_numeric(cur["intended concurrency"], errors="coerce") + base["intended concurrency"] = pd.to_numeric(base["intended concurrency"], errors="coerce") + + common_conc = set(cur["intended concurrency"].dropna().unique()) & set( + base["intended concurrency"].dropna().unique() + ) + common_conc.discard(1) + + if not common_conc: + continue + + cur_common = cur[cur["intended concurrency"].isin(common_conc)] + base_common = base[base["intended concurrency"].isin(common_conc)] + + for col, meta in metrics.items(): + if col not in cur_common.columns or col not in base_common.columns: + continue + + cur_vals = pd.to_numeric(cur_common[col], errors="coerce").dropna() + base_vals = pd.to_numeric(base_common[col], errors="coerce").dropna() + + cur_gm = geometric_mean(cur_vals) + base_gm = geometric_mean(base_vals) + + if cur_gm is None or base_gm is None: + continue + + pct_diff = ((cur_gm - base_gm) / base_gm) * 100 + threshold = meta["threshold"] + + if meta["higher_is_better"]: + is_regression = pct_diff < -threshold + is_improvement = pct_diff > threshold + else: + is_regression = pct_diff > threshold + is_improvement = pct_diff < -threshold + + results.append( + RegressionResult( + profile=profile, + metric_column=col, + metric_label=meta["label"], + current_value=cur_gm, + baseline_value=base_gm, + pct_diff=pct_diff, + is_regression=is_regression, + is_improvement=is_improvement, + ) + ) + + regressions = [r for r in results if r.is_regression] + logger.info( + "Comparison complete: %d metric comparisons, %d regressions", len(results), len(regressions) + ) + return results, "" + + +def run_regression_analysis( + current_csv_path: Path, + consolidated_csv_path: Path, + compare_version: str, + current_version: str, + output_file: Path, + profile_map: dict[tuple[int, int], str], + metrics: dict[str, dict[str, Any]], + restrict_profiles: list[str] | None = None, +) -> dict[str, Any]: + """Run regression analysis comparing current run against baseline. + + Returns dict with status, regressions, improvements, and skip_reason. + """ + try: + results, skip_reason = compare_runs( + str(current_csv_path), + str(consolidated_csv_path), + compare_version, + current_version, + profile_map=profile_map, + metrics=metrics, + restrict_profiles=restrict_profiles, + ) + + if skip_reason: + analysis = { + "status": "skipped", + "reason": skip_reason, + "current_version": current_version, + "compare_version": compare_version, + } + else: + regressions = [r for r in results if r.is_regression] + improvements = [r for r in results if r.is_improvement] + + analysis = { + "status": "completed", + "current_version": current_version, + "compare_version": compare_version, + "total_comparisons": len(results), + "regression_count": len(regressions), + "improvement_count": len(improvements), + "regressions": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in regressions + ], + "improvements": [ + { + "profile": r.profile, + "metric": r.metric_label, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + } + for r in improvements + ], + "all_results": [ + { + "profile": r.profile, + "metric": r.metric_label, + "metric_column": r.metric_column, + "current": round(r.current_value, 2), + "baseline": round(r.baseline_value, 2), + "pct_diff": round(r.pct_diff, 1), + "is_regression": r.is_regression, + "is_improvement": r.is_improvement, + } + for r in results + ], + } + + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(analysis, f, indent=2) + + return analysis + + except Exception as e: + logger.exception("Regression analysis failed") + return {"status": "failed", "error": str(e)} + + +# --------------------------------------------------------------------------- +# RHAIIS constants +# --------------------------------------------------------------------------- + +PROFILE_MAP = { + (1000, 1000): "profile1", + (512, 2048): "profile2", + (2048, 128): "profile3", + (8000, 1000): "profile4", +} + +METRICS = { + "total_tok/sec": {"label": "Total Throughput", "higher_is_better": True, "threshold": 5}, + "output_tok/sec": {"label": "Output Throughput", "higher_is_better": True, "threshold": 5}, + "ttft_p95": {"label": "TTFT P95", "higher_is_better": False, "threshold": 10}, + "itl_p95": {"label": "ITL P95", "higher_is_better": False, "threshold": 5}, + "request_latency_median": { + "label": "Median E2E Latency", + "higher_is_better": False, + "threshold": 5, + }, +} + +DASHBOARD_BASE_URL = "https://staging-aidash.apps.ocp4.intlab.redhat.com/" + +PROFILE_DISPLAY_NAMES = { + "profile1": "Profile A: Balanced (1k/1k)", + "profile2": "Profile B: Variable Workload (512/2k)", + "profile3": "Profile C: Large Prompt (2k/128)", + "profile4": "Profile D: Prefill Heavy (8k/1k)", +} + + +def _build_dashboard_url( + *, + model: str = "", + accelerator: str = "", + current_version: str = "", + compare_version: str = "", + profiles: list[str] | None = None, + tp: str = "", +) -> str: + """Build a RHAIIS dashboard URL with filters pre-selected.""" + from urllib.parse import quote, urlencode + + params: dict[str, str] = {"view": "RHAIIS Dashboard"} + + if accelerator: + params["accelerators"] = accelerator + if model: + params["models"] = model + + versions = ",".join(v for v in [current_version, compare_version] if v) + if versions: + params["versions"] = versions + + if profiles: + display_names = [PROFILE_DISPLAY_NAMES.get(p, p) for p in profiles] + params["profile"] = ",".join(display_names) + + if tp: + try: + params["tp_sizes"] = f"{float(tp):.1f}" + except (ValueError, TypeError): + params["tp_sizes"] = tp + + params["section"] = "performance_plots" + + return f"{DASHBOARD_BASE_URL}?{urlencode(params, quote_via=quote)}" + + +def send_regression_notification( + analysis_result: dict, + *, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + notification_vault: str | None = None, + dry_run: bool = False, + report_url: str = "", + tp: str = "", + dp: str = "", +) -> bool: + """Send regression/improvement Slack notification from analysis results. + + Args: + analysis_result: Output from run_regression_analysis() + model: Model name for display + accelerator: Accelerator name for display + job_id: Job identifier + slack_user: Slack user ID (e.g. U01ABC123) to @-mention, or display name + notification_vault: Vault containing topsail-bot.slack-token + dry_run: Log only, don't send + report_url: Optional presigned URL to an agent analysis report + tp: Tensor parallelism size for display + dp: Data parallelism size for display + + Returns: + True if notification sent successfully + """ + status = analysis_result.get("status") + if status == "skipped": + reason = analysis_result.get("reason", "unknown") + logger.info("Regression analysis skipped: %s", reason) + return True + + if status != "completed": + return True + + regressions = analysis_result.get("regressions", []) + improvements = analysis_result.get("improvements", []) + + if not regressions and not improvements: + logger.info("No regressions or improvements to report") + return True + + current_version = analysis_result.get("current_version", "") + compare_version = analysis_result.get("compare_version", "") + + if regressions and improvements: + icon = ":warning:" + headline = "Performance regressions and improvements detected" + elif regressions: + icon = ":warning:" + headline = "Performance regressions detected" + else: + icon = ":large_green_circle:" + headline = "Performance improvements detected" + + detail_lines = [] + all_results = analysis_result.get("all_results", []) + profiles = sorted( + {r["profile"] for r in all_results if r.get("is_regression") or r.get("is_improvement")} + ) + + for profile in profiles: + profile_results = [ + r + for r in all_results + if r["profile"] == profile and (r.get("is_regression") or r.get("is_improvement")) + ] + detail_lines.append(f"\n*{profile}:*") + for r in profile_results: + if r.get("is_regression"): + direction = "dropped" if r["pct_diff"] < 0 else "increased" + detail_lines.append( + f" :red_circle: *{r['metric']}*: {direction} {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + else: + detail_lines.append( + f" :large_green_circle: *{r['metric']}*: improved {abs(r['pct_diff']):.1f}% " + f"({r['baseline']:.2f} \u2192 {r['current']:.2f})" + ) + + details = "\n".join(detail_lines) + + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" + + parallelism_line = "" + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + if parallelism_parts: + parallelism_line = f"*Parallelism:* {', '.join(parallelism_parts)}\n" + + dashboard_url = _build_dashboard_url( + model=model, + accelerator=accelerator, + current_version=current_version, + compare_version=compare_version, + profiles=profiles if profiles else None, + tp=tp, + ) + dashboard_line = f"*Dashboard:* <{dashboard_url}|View Dashboard>\n" + + message = ( + f"{icon} *{headline}*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"*Versions:* {current_version} vs {compare_version} (baseline)\n" + f"{report_line}" + f"{dashboard_line}" + f"*Changes:*\n{details}" + ) + + if dry_run: + logger.info("DRY RUN regression notification:\n%s", message) + return True + + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) + + +def send_failure_notification( + *, + error: str, + model: str = "", + accelerator: str = "", + job_id: str = "", + slack_user: str = "", + notification_vault: str | None = None, + dry_run: bool = False, + tp: str = "", + dp: str = "", + version: str = "", + workload_keys: list[str] | None = None, + cluster: str = "", +) -> bool: + """Send a Slack alert when the RHAIIS pipeline fails. + + Args: + error: Error message or traceback summary + model: Model name for display + accelerator: Accelerator name for display + job_id: FournosJob name + slack_user: Slack user ID to @-mention + notification_vault: Vault containing topsail-bot.slack-token + dry_run: Log only, don't send + tp: Tensor parallelism size + dp: Data parallelism size + version: vLLM / RHAIIS version string + workload_keys: List of workload profile keys + cluster: Cluster name the job ran on + + Returns: + True if notification sent successfully + """ + if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): + user_line = f"*Triggered by:* <@{slack_user}>\n" + elif slack_user: + user_line = f"*Triggered by:* {slack_user}\n" + else: + user_line = "" + + parallelism_parts = [] + if tp: + parallelism_parts.append(f"TP={tp}") + if dp: + parallelism_parts.append(f"DP={dp}") + parallelism_line = ( + f"*Parallelism:* {', '.join(parallelism_parts)}\n" if parallelism_parts else "" + ) + + profiles_line = "" + if workload_keys: + profiles_line = f"*Workloads:* {', '.join(workload_keys)}\n" + + cluster_line = f"*Cluster:* {cluster}\n" if cluster else "" + version_line = f"*Version:* {version}\n" if version else "" + + error_text = error if len(error) <= 500 else error[:500] + "..." + + message = ( + f":x: *RHAIIS Pipeline Failed*\n" + f"{user_line}" + f"*Job:* `{job_id}`\n" + f"*Model:* {model}\n" + f"*Accelerator:* {accelerator}\n" + f"{parallelism_line}" + f"{version_line}" + f"{cluster_line}" + f"{profiles_line}" + f"*Error:*\n```{error_text}```" + ) + + if dry_run: + logger.info("DRY RUN failure notification:\n%s", message) + return True + + return _send_via_topsail_bot( + message, notification_vault=notification_vault, channel_id=RHAIIS_SLACK_CHANNEL_ID + ) diff --git a/projects/rhaiis/postprocess/s3_dashboard.py b/projects/rhaiis/postprocess/s3_dashboard.py new file mode 100644 index 000000000..060288284 --- /dev/null +++ b/projects/rhaiis/postprocess/s3_dashboard.py @@ -0,0 +1,207 @@ +"""S3 dashboard CSV sync and profiler trace upload for rhaiis.""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +def sync_csv_to_s3( + csv_path: Path, + *, + s3_bucket: str, + s3_key: str, + vault_name: str, + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Download consolidated CSV from S3, append new rows, re-upload.""" + try: + import pandas as pd + except ImportError: + return {"status": "failed", "error": "pandas not available"} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + if dry_run: + logger.info("DRY RUN: Would sync %s to s3://%s/%s", csv_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + consolidated_path = None + try: + s3 = create_s3_client(credentials_path) + + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: + consolidated_path = tmp.name + + try: + s3.download_file(s3_bucket, s3_key, consolidated_path) + logger.info("Downloaded s3://%s/%s", s3_bucket, s3_key) + except Exception as e: + if hasattr(e, "response") and e.response.get("Error", {}).get("Code") == "404": + logger.warning("Consolidated CSV not found on S3, uploading new CSV") + s3.upload_file(str(csv_path), s3_bucket, s3_key) + return {"status": "success", "action": "created", "rows_added": -1} + raise + + new_df = pd.read_csv(csv_path) + if new_df.empty: + logger.warning("No data rows in generated CSV %s", csv_path) + return {"status": "skipped", "reason": "empty CSV"} + + existing_df = pd.read_csv(consolidated_path, on_bad_lines="warn") + merged_df = pd.concat([existing_df, new_df], ignore_index=True) + merged_df.to_csv(consolidated_path, index=False) + rows_added = len(new_df) + logger.info("Appended %d rows (%d total)", rows_added, len(merged_df)) + + s3.upload_file(consolidated_path, s3_bucket, s3_key) + logger.info("Uploaded consolidated CSV to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "action": "appended", "rows_added": rows_added} + + except Exception as e: + logger.error("S3 CSV sync failed: %s", e) + return {"status": "failed", "error": str(e)} + finally: + import os + + if consolidated_path and os.path.exists(consolidated_path): + os.unlink(consolidated_path) + + +def upload_profiler_traces_to_s3( + traces_dir: Path, + *, + model_name: str, + accelerator: str, + tp_size: int, + version: str, + profile_labels: list[str], + s3_bucket: str, + s3_prefix: str, + vault_name: str, + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload profiler traces to S3. + + Path: s3://{bucket}/{prefix}/{accel}/{model}/tp{N}/{version}/{profile_label}/ + """ + if not traces_dir.exists(): + return {"status": "skipped", "reason": f"traces dir not found: {traces_dir}"} + + trace_files = [ + f + for f in traces_dir.iterdir() + if f.name.startswith("trace") and "rank0" in f.name and f.suffix in (".json", ".gz") + ] + if not trace_files: + return {"status": "skipped", "reason": "no trace files found"} + + if dry_run: + logger.info("DRY RUN: Would upload %d traces to S3", len(trace_files)) + return {"status": "success", "dry_run": True, "trace_count": len(trace_files)} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + accel_folder = _accelerator_s3_folder(accelerator) + model_encoded = model_name.replace("/", "--") + uploaded = 0 + + try: + s3 = create_s3_client(credentials_path) + + for trace_file in trace_files: + matched_label = None + for label in profile_labels: + if label in trace_file.name: + matched_label = label + break + if not matched_label: + matched_label = profile_labels[0] if len(profile_labels) == 1 else None + if not matched_label: + logger.warning("Could not match trace %s to a profile label", trace_file.name) + continue + + s3_key = ( + f"{s3_prefix}/{accel_folder}/{model_encoded}" + f"/tp{tp_size}/{version}/{matched_label}/{trace_file.name}" + ) + + try: + s3.head_object(Bucket=s3_bucket, Key=s3_key) + s3.copy_object( + Bucket=s3_bucket, + CopySource={"Bucket": s3_bucket, "Key": s3_key}, + Key=s3_key + ".bak", + ) + logger.info("Backed up existing trace at s3://%s/%s", s3_bucket, s3_key) + except Exception: + pass + + s3.upload_file(str(trace_file), s3_bucket, s3_key) + logger.info("Uploaded trace to s3://%s/%s", s3_bucket, s3_key) + uploaded += 1 + + return {"status": "success", "uploaded": uploaded} + except Exception as e: + logger.error("Trace upload failed: %s", e) + return {"status": "failed", "error": str(e), "uploaded": uploaded} + + +def upload_predictor_log_to_s3( + log_path: Path, + *, + run_uuid: str, + s3_bucket: str, + vault_name: str, + credentials_file: str = "aws.credentials", + dry_run: bool = False, +) -> dict[str, Any]: + """Upload the vLLM predictor pod log to S3 as ``logs/{run_uuid}.log``.""" + if not log_path.exists(): + return {"status": "skipped", "reason": f"log not found: {log_path}"} + + s3_key = f"logs/{run_uuid}.log" + + if dry_run: + logger.info("DRY RUN: Would upload %s to s3://%s/%s", log_path, s3_bucket, s3_key) + return {"status": "success", "dry_run": True} + + from projects.caliper.cli.s3_export import create_s3_client, get_aws_credentials + + credentials_path = get_aws_credentials(vault_name, credentials_file) + if not credentials_path: + return {"status": "failed", "error": f"AWS credentials not found in vault {vault_name}"} + + try: + s3 = create_s3_client(credentials_path) + s3.upload_file(str(log_path), s3_bucket, s3_key) + logger.info("Uploaded predictor log to s3://%s/%s", s3_bucket, s3_key) + return {"status": "success", "s3_key": s3_key} + except Exception as e: + logger.error("Predictor log upload failed: %s", e) + return {"status": "failed", "error": str(e)} + + +def _accelerator_s3_folder(accelerator: str) -> str: + upper = accelerator.upper() + if upper.startswith("H200"): + return "H200" + if upper.startswith("B200"): + return "B200" + if upper.startswith("MI300"): + return "MI300x" + return accelerator diff --git a/projects/rhaiis/toolbox/copy_profiler_traces/main.py b/projects/rhaiis/toolbox/copy_profiler_traces/main.py new file mode 100644 index 000000000..147fe8dfe --- /dev/null +++ b/projects/rhaiis/toolbox/copy_profiler_traces/main.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str): + return execute_tasks(locals()) + + +@task +def setup_directories(args, context): + shell.mkdir("artifacts/traces") + return "Traces directory created" + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def list_trace_files(args, context): + result = shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + "-- sh -c 'ls /tmp/trace_*.json* 2>/dev/null || echo NO_TRACES'", + check=False, + log_stdout=False, + ) + + if "NO_TRACES" in result.stdout or not result.stdout.strip(): + raise RuntimeError(f"No profiler traces found in pod {context.pod_name}") + + trace_list = result.stdout.strip() + context.trace_count = len(trace_list.splitlines()) + return f"Found {context.trace_count} trace files" + + +@task +def copy_traces(args, context): + traces_dir = args.artifact_dir / "artifacts/traces" + shell.run( + 'bash -o pipefail -c "' + f"oc exec {context.pod_name} -n {args.namespace}" + " -- sh -c 'cd /tmp && tar cf - trace_*.json*'" + f' | tar --no-same-owner -xf - -C {traces_dir}"', + ) + copied = list(traces_dir.glob("trace_*.json*")) + return f"Copied {len(copied)} trace files to {traces_dir}" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py index c8cd621d9..add66bb25 100644 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py +++ b/projects/rhaiis/toolbox/deploy_kserve_isvc/main.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 +from __future__ import annotations + +import yaml + from projects.core.dsl import ( entrypoint, execute_tasks, task, - template, ) from projects.core.dsl.utils.k8s import oc, oc_resource_exists @@ -12,42 +15,18 @@ @entrypoint def run( *, - deployment_name: str, namespace: str, - model_id: str, - vllm_image: str, - accelerator: str = "nvidia", - vllm_args: dict | None = None, - env_vars: dict | None = None, - replicas: int = 1, - cpu_request: str = "4", - memory_request: str = "16Gi", - storage_source: str = "hf", - storage_pvc: str = "", - image_pull_secret: str = "", - service_account_name: str = "", + servingruntime_file: str, + inferenceservice_file: str, ): - return execute_tasks(locals()) + """Deploy a KServe InferenceService from pre-rendered YAML files. - -@task -def prepare_args(args, context): - context.env_vars_list = [] - for key, value in (args.env_vars or {}).items(): - context.env_vars_list.append({"name": key, "value": str(value)}) - - vllm_args = args.vllm_args or {} - tp_size = vllm_args.get("tensor-parallel-size", 1) - context.gpu_count = int(tp_size) - - if args.storage_pvc: - context.use_pvc = True - context.pvc_name = args.storage_pvc - else: - context.use_pvc = False - context.pvc_name = "" - - return f"GPU count={context.gpu_count}, env_vars={len(context.env_vars_list)}, pvc={context.use_pvc}" + Args: + namespace: Target namespace for the deployment. + servingruntime_file: Path to a ServingRuntime YAML file. + inferenceservice_file: Path to an InferenceService YAML file. + """ + return execute_tasks(locals()) @task @@ -59,19 +38,19 @@ def ensure_namespace(args, context): @task -def render_and_apply_servingruntime(args, context): - output_path = args.artifact_dir / "servingruntime.yaml" - template.render_template_to_file("servingruntime.yaml.j2", output_path) - oc("apply", "-f", str(output_path)) - return f"Applied ServingRuntime {args.deployment_name}" +def apply_servingruntime(args, context): + oc("apply", "-f", args.servingruntime_file) + with open(args.servingruntime_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") + return f"Applied ServingRuntime {name}" @task -def render_and_apply_inferenceservice(args, context): - output_path = args.artifact_dir / "inferenceservice.yaml" - template.render_template_to_file("inferenceservice.yaml.j2", output_path) - oc("apply", "-f", str(output_path)) - return f"Applied InferenceService {args.deployment_name}" +def apply_inferenceservice(args, context): + oc("apply", "-f", args.inferenceservice_file) + with open(args.inferenceservice_file) as f: + name = yaml.safe_load(f).get("metadata", {}).get("name", "") + return f"Applied InferenceService {name}" if __name__ == "__main__": diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 deleted file mode 100644 index baf29eadd..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/inferenceservice.yaml.j2 +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - serving.kserve.io/enable-prometheus-scraping: "true" - storage.kserve.io/readonly: "false" - prometheus.io/scrape: "true" - prometheus.io/path: /metrics - prometheus.io/port: "8080" - labels: - opendatahub.io/dashboard: "true" - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: - predictor: - minReplicas: {{ args.replicas }} - maxReplicas: {{ args.replicas }} - model: - modelFormat: - name: vLLM - runtime: {{ args.deployment_name }} - resources: -{%- if args.accelerator == "nvidia" %} - limits: - nvidia.com/gpu: "{{ ctx.gpu_count }}" - requests: - nvidia.com/gpu: "{{ ctx.gpu_count }}" -{%- elif args.accelerator == "amd" %} - limits: - amd.com/gpu: "{{ ctx.gpu_count }}" - requests: - amd.com/gpu: "{{ ctx.gpu_count }}" -{%- endif %} - cpu: "{{ args.cpu_request }}" - memory: "{{ args.memory_request }}" -{%- if args.storage_source == "hf" and ctx.use_pvc %} - storageUri: pvc://{{ ctx.pvc_name }}/ -{%- elif args.storage_source != "hf" %} - storageUri: {{ args.storage_source }}://{{ args.model_id }} -{%- endif %} -{%- if args.service_account_name %} - serviceAccountName: {{ args.service_account_name }} -{%- endif %} diff --git a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 b/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 deleted file mode 100644 index baabd0852..000000000 --- a/projects/rhaiis/toolbox/deploy_kserve_isvc/templates/servingruntime.yaml.j2 +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: serving.kserve.io/v1alpha1 -kind: ServingRuntime -metadata: - annotations: - opendatahub.io/template-display-name: ServingRuntime for vLLM | RHAIIS - labels: - opendatahub.io/dashboard: "true" - name: {{ args.deployment_name }} - namespace: {{ args.namespace }} -spec: -{%- if args.image_pull_secret %} - imagePullSecrets: - - name: {{ args.image_pull_secret }} -{%- endif %} - builtInAdapter: - modelLoadingTimeoutMillis: 300000 - multiModel: false - supportedModelFormats: - - autoSelect: true - name: vLLM - containers: - - name: kserve-container - image: "{{ args.vllm_image }}" - command: - - python3 - - -m - - vllm.entrypoints.openai.api_server - args: -{%- if args.storage_source == "hf" %} - - --model={{ args.model_id }} -{%- else %} - - --model=/mnt/models - - --served-model-name={{ args.model_id }} -{%- endif %} - - --port=8080 -{%- for key, val in (args.vllm_args or {}).items() %} - {%- if val is boolean %} - {%- if val %} - - --{{ key }} - {%- endif %} - {%- else %} - - --{{ key }}={{ val }} - {%- endif %} -{%- endfor %} - env: -{%- if args.storage_source == "hf" %} - - name: HF_HUB_OFFLINE - value: "0" - - name: HOME - value: /mnt/models - - name: HF_HOME - value: /mnt/models - - name: VLLM_CACHE_DIR - value: /mnt/models/.cache/vllm - - name: HF_DATASETS_CACHE - value: /mnt/models/.cache/huggingface/datasets - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: storage-config - key: HF_TOKEN -{%- endif %} -{%- for env_item in ctx.env_vars_list %} - - name: {{ env_item.name }} - value: "{{ env_item.value }}" -{%- endfor %} - ports: - - containerPort: 8080 - protocol: TCP -{%- if ctx.gpu_count > 1 %} - volumeMounts: - - name: shared-memory - mountPath: /dev/shm - volumes: - - name: shared-memory - emptyDir: - medium: Memory - sizeLimit: 8Gi -{%- endif %} diff --git a/projects/rhaiis/toolbox/enable_profiler_gate/main.py b/projects/rhaiis/toolbox/enable_profiler_gate/main.py new file mode 100644 index 000000000..c370953ed --- /dev/null +++ b/projects/rhaiis/toolbox/enable_profiler_gate/main.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + + +@entrypoint +def run(*, name: str, namespace: str, gate_value: str = "0", disable: bool = False): + """Toggle the vLLM profiler gate file on a KServe predictor pod. + + Writes (or removes) /tmp/profiler_gate inside the predictor container + to enable or disable on-demand profiling for a running inference service. + + Args: + name: KServe InferenceService name used to locate the predictor pod. + namespace: Kubernetes namespace where the InferenceService is deployed. + gate_value: Value written to the gate file (default "0"). + disable: When True, removes the gate file instead of creating it. + """ + return execute_tasks(locals()) + + +@task +def find_predictor_pod(args, context): + result = shell.run( + f"oc get pod -oname " + f"-lserving.kserve.io/inferenceservice={args.name} " + f"-n {args.namespace} " + "| head -1", + check=False, + ) + pod_name = result.stdout.strip() + if not pod_name: + raise RuntimeError(f"No predictor pod found for {args.name} in {args.namespace}") + context.pod_name = pod_name + return f"Found pod: {pod_name}" + + +@task +def toggle_profiler_gate(args, context): + if args.disable: + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} -- sh -c 'rm -f /tmp/profiler_gate'", + check=False, + ) + return f"Profiler gate disabled on {context.pod_name}" + + shell.run( + f"oc exec {context.pod_name} -n {args.namespace} " + f"-- sh -c 'echo {args.gate_value} > /tmp/profiler_gate'", + ) + return f"Profiler gate enabled on {context.pod_name} (value={args.gate_value})" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py new file mode 100644 index 000000000..2fd23542d --- /dev/null +++ b/projects/rhaiis/toolbox/verify_profiler_prereqs/main.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Verify that the vLLM profiler webhook infrastructure is ready. + +Ported from model-furnace's oc_commands.verify_profiler_prerequisites(). +Checks webhook pod, MutatingWebhookConfiguration, TLS secret, and +ConfigMap before any profiler run is attempted. +""" + +from projects.core.dsl import ( + entrypoint, + execute_tasks, + shell, + task, +) + +MIN_CA_BUNDLE_LENGTH = 100 + + +@entrypoint +def run(*, namespace: str): + return execute_tasks(locals()) + + +@task +def check_webhook_pod(args, context): + result = shell.run( + "oc get pods -n vllm-profiler -l app=env-injector -o jsonpath='{.items[0].status.phase}'", + check=False, + ) + phase = result.stdout.strip().strip("'") + if result.returncode != 0 or phase != "Running": + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook pod not running (status: {phase or 'not found'}). " + "Deploy it with: ./scripts/deploy.sh" + ) + return f"FAIL: webhook pod status={phase or 'not found'}" + return "OK: webhook pod is Running" + + +@task +def check_webhook_target_namespace(args, context): + result = shell.run( + "oc get deployment env-injector -n vllm-profiler " + "-o jsonpath='{.spec.template.spec.containers[0]" + '.env[?(@.name=="TARGET_NAMESPACE")].value}\'', + check=False, + ) + target_ns = result.stdout.strip().strip("'") + if result.returncode != 0 or not target_ns: + context.errors = getattr(context, "errors", []) + context.errors.append("Could not read webhook TARGET_NAMESPACE from deployment") + return "FAIL: could not read TARGET_NAMESPACE" + if target_ns != args.namespace: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"Webhook TARGET_NAMESPACE is '{target_ns}' but deployment namespace " + f"is '{args.namespace}'. Update manifests.yaml and run: " + "oc apply -f manifests.yaml" + ) + return f"FAIL: TARGET_NAMESPACE={target_ns} != {args.namespace}" + return f"OK: TARGET_NAMESPACE matches '{args.namespace}'" + + +@task +def check_mutating_webhook(args, context): + result = shell.run( + "oc get mutatingwebhookconfiguration env-injector-webhook " + "-o jsonpath='{.webhooks[0].clientConfig.caBundle}'", + check=False, + ) + ca_bundle = result.stdout.strip().strip("'") + if result.returncode != 0 or len(ca_bundle) < MIN_CA_BUNDLE_LENGTH: + context.errors = getattr(context, "errors", []) + context.errors.append( + "MutatingWebhookConfiguration missing or caBundle is empty. " + "Run: bash scripts/gen-certs.sh && bash scripts/patch-ca-bundle.sh" + ) + return "FAIL: MutatingWebhookConfiguration missing or bad caBundle" + return "OK: MutatingWebhookConfiguration has valid caBundle" + + +@task +def check_tls_secret(args, context): + result = shell.run( + "oc get secret env-injector-certs -n vllm-profiler -o name", + check=False, + ) + if result.returncode != 0: + context.errors = getattr(context, "errors", []) + context.errors.append( + "TLS secret 'env-injector-certs' not found in vllm-profiler namespace" + ) + return "FAIL: TLS secret missing" + return "OK: TLS secret exists" + + +@task +def check_configmap(args, context): + result = shell.run( + f"oc get configmap env-injector-files -n {args.namespace} -o jsonpath='{{.data}}'", + check=False, + ) + if result.returncode != 0 or "sitecustomize.py" not in result.stdout: + context.errors = getattr(context, "errors", []) + context.errors.append( + f"ConfigMap 'env-injector-files' not found in {args.namespace}. " + "Create it with: oc apply -k " + ) + return f"FAIL: ConfigMap missing in {args.namespace}" + return f"OK: ConfigMap exists in {args.namespace}" + + +@task +def raise_on_errors(args, context): + errors = getattr(context, "errors", []) + if errors: + raise ValueError("Profiler prerequisites not met:\n - " + "\n - ".join(errors)) + return "All profiler prerequisites verified" + + +if __name__ == "__main__": + run.main() diff --git a/projects/rhoai/library/deploy.py b/projects/rhoai/library/deploy.py new file mode 100644 index 000000000..cefc7b28f --- /dev/null +++ b/projects/rhoai/library/deploy.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import base64 +import json +import logging +import tempfile +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from projects.cluster.toolbox.cluster_deploy_operator import main as cluster_deploy_operator +from projects.cluster.toolbox.deploy_custom_catalog import main as deploy_custom_catalog +from projects.cluster.toolbox.wait_for_crds import main as wait_for_crds_command +from projects.core.dsl.utils.k8s import oc, oc_get_json +from projects.core.library import vault + +logger = logging.getLogger(__name__) + +RHOAI_PULL_SECRET_NAMESPACE = "openshift-config" +RHOAI_PULL_SECRET_NAME = "pull-secret" +RHOAI_CATALOG_REGISTRIES = ("quay.io/rhoai",) +RHOAI_REGISTRIES = ( + "registry.stage.redhat.io/rhaii", + "registry.stage.redhat.io/rhaii-early-access", +) + + +class _RhoaiCustomCatalogPullSecretVaultConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + content: str + + +class _RhoaiCustomCatalogPullSecretConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + vault: _RhoaiCustomCatalogPullSecretVaultConfig + + +class _RhoaiCustomCatalogPullSecretInput(BaseModel): + model_config = ConfigDict(extra="ignore") + + pull_secret: _RhoaiCustomCatalogPullSecretConfig + + +class RhoaiCustomCatalogConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + enabled: bool = False + name: str + namespace: str + image: str | None = None + display_name: str | None = None + publisher: str | None = None + pull_secret: _RhoaiCustomCatalogPullSecretConfig + staging_pull_secret: _RhoaiCustomCatalogPullSecretConfig | None = None + + +class RhoaiOperatorConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + + custom_catalog: RhoaiCustomCatalogConfig + namespace: str + datasciencecluster_name: str + components: list[str] = Field(default_factory=list) + required_crds_before_dsc: list[str] = Field(default_factory=list) + required_crds_after_dsc: list[str] = Field(default_factory=list) + + +def custom_catalog_pull_secret_path(custom_catalog: dict[str, Any]) -> Path: + catalog = _RhoaiCustomCatalogPullSecretInput.model_validate(custom_catalog) + secret_path = vault.get_vault_content_path( + catalog.pull_secret.vault.name, + catalog.pull_secret.vault.content, + ) + if secret_path is None: + raise RuntimeError( + f"RHOAI custom catalog pull secret is required but not available. " + f"Vault '{catalog.pull_secret.vault.name}' with content " + f"'{catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." + ) + return secret_path + + +def operator_spec_by_package(platform: dict[str, Any], package: str) -> dict[str, Any]: + operators = platform["operators"] + if isinstance(operators, dict): + if package in operators: + return {"package": package, **operators[package]} + raise KeyError(f"Unknown operator package: {package}") + + for operator_spec in operators: + if operator_spec["package"] == package: + return operator_spec + raise KeyError(f"Unknown operator package: {package}") + + +def ensure_operator_subscription(operator_spec: dict[str, str]) -> dict[str, object]: + return cluster_deploy_operator.run( + package_name=operator_spec["package"], + target_namespace=operator_spec["namespace"], + source_name=operator_spec["source"], + channel=operator_spec["channel"], + source_namespace=operator_spec.get("source_namespace", "openshift-marketplace"), + display_name=operator_spec.get("display_name", operator_spec["package"]), + artifact_dirname_suffix=f"_{operator_spec['package']}", + ) + + +def deploy_rhoai_custom_catalog(*, custom_catalog: RhoaiCustomCatalogConfig) -> int: + if not custom_catalog.enabled: + logger.info("RHOAI custom catalog disabled; using default catalog source") + return 0 + + if not custom_catalog.image: + raise RuntimeError("RHOAI custom catalog is enabled but no image was configured") + + return deploy_custom_catalog.run( + catalog_source_name=custom_catalog.name, + catalog_namespace=custom_catalog.namespace, + catalog_image=custom_catalog.image, + display_name=custom_catalog.display_name or custom_catalog.name, + publisher=custom_catalog.publisher or "", + artifact_dirname_suffix="rhoai", + ) + + +def rhoai_operator_spec( + *, + custom_catalog: RhoaiCustomCatalogConfig, + operator_spec: dict[str, str], +) -> dict[str, str]: + if not custom_catalog.enabled: + return operator_spec + + updated_spec = dict(operator_spec) + updated_spec["source"] = custom_catalog.name + updated_spec["source_namespace"] = custom_catalog.namespace + return updated_spec + + +def prepare_rhcl_operator(platform: dict[str, Any]) -> None: + operator_spec = operator_spec_by_package(platform, "rhcl-operator") + ensure_operator_subscription(operator_spec) + + +def _decode_pull_secret(secret_data: dict[str, Any]) -> str: + encoded = secret_data.get("data", {}).get(".dockerconfigjson") + if not encoded: + raise RuntimeError("openshift-config/pull-secret is missing .dockerconfigjson data") + + return base64.b64decode(encoded).decode("utf-8") + + +def _read_vault_pull_secret(pull_secret_path: Path) -> str | dict[str, Any]: + raw_content = pull_secret_path.read_text(encoding="utf-8").strip() + if not raw_content: + raise RuntimeError(f"RHOAI pull secret file is empty: {pull_secret_path}") + + try: + parsed = json.loads(raw_content) + except json.JSONDecodeError: + return raw_content + + if isinstance(parsed, dict) and "auths" in parsed: + return parsed + + return raw_content + + +def _apply_vault_pull_secret( + *, + current_secret: dict[str, Any], + pull_secret_path: Path, + registries: tuple[str, ...], + temp_path: Path, +) -> dict[str, Any]: + pull_secret_payload = _read_vault_pull_secret(pull_secret_path) + + if isinstance(pull_secret_payload, dict): + merged_secret = _merge_pull_secret_auths(current_secret, pull_secret_payload) + temp_path.write_text(json.dumps(merged_secret, indent=2, sort_keys=True), encoding="utf-8") + return merged_secret + + for registry in registries: + oc( + "registry", + "login", + f"--registry={registry}", + f"--auth-basic={pull_secret_payload}", + "--to", + str(temp_path), + ) + return json.loads(temp_path.read_text(encoding="utf-8")) + + +def _current_pull_secret_json() -> dict[str, Any]: + current_secret = oc_get_json( + "secret", + name=RHOAI_PULL_SECRET_NAME, + namespace=RHOAI_PULL_SECRET_NAMESPACE, + ) + decoded_secret = _decode_pull_secret(current_secret) + try: + return json.loads(decoded_secret) + except json.JSONDecodeError as exc: + raise RuntimeError("openshift-config/pull-secret is not valid dockerconfigjson") from exc + + +def _normalize_registry_key(registry: str) -> str: + return registry.removeprefix("https://").removeprefix("http://").rstrip("/") + + +def _auth_entry_covers_registry(auth_key: str, required_registry: str) -> bool: + auth_key = _normalize_registry_key(auth_key) + required_registry = _normalize_registry_key(required_registry) + return auth_key == required_registry or required_registry.startswith(f"{auth_key}/") + + +def _registries_present( + pull_secret: dict[str, Any], registries: tuple[str, ...] = RHOAI_REGISTRIES +) -> bool: + auth_keys = tuple(pull_secret.get("auths", {}).keys()) + return all( + any(_auth_entry_covers_registry(auth_key, registry) for auth_key in auth_keys) + for registry in registries + ) + + +def _merge_pull_secret_auths( + current_secret: dict[str, Any], pull_secret_payload: dict[str, Any] +) -> dict[str, Any]: + merged_secret = dict(current_secret) + merged_auths = dict(current_secret.get("auths", {})) + merged_auths.update(pull_secret_payload.get("auths", {})) + merged_secret["auths"] = merged_auths + return merged_secret + + +def wait_for_rhoai_pull_secret_ready( + *, timeout_seconds: int = 600, poll_interval_seconds: int = 15 +) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: str | None = None + + while time.monotonic() < deadline: + try: + pull_secret = _current_pull_secret_json() + if not _registries_present(pull_secret, (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES)): + raise RuntimeError("required RHOAI registries are not yet present in pull secret") + mcp_status = oc( + "get", + "mcp", + "-o", + r'jsonpath={.items[*].status.conditions[?(@.type=="Updated")].status}', + check=False, + log_stdout=False, + log_stderr=False, + ).stdout.strip() + if not mcp_status or "False" in mcp_status: + raise RuntimeError("machine config pools are still updating") + + logger.info( + "RHOAI pull secret now contains %s and machine config pools are updated", + ", ".join(RHOAI_REGISTRIES), + ) + return + except RuntimeError as exc: + last_error = str(exc) + logger.info("Waiting for RHOAI pull secret propagation: %s", last_error) + time.sleep(poll_interval_seconds) + + raise RuntimeError( + f"Timed out waiting for RHOAI registries pull secret propagation: {last_error}" + ) + + +def prepare_rhoai_pull_secret(custom_catalog: RhoaiCustomCatalogConfig) -> None: + pull_secret_path = vault.get_vault_content_path( + custom_catalog.pull_secret.vault.name, + custom_catalog.pull_secret.vault.content, + ) + if pull_secret_path is None: + raise RuntimeError( + f"RHOAI custom catalog is enabled, but the required pull secret is not available. " + f"Vault '{custom_catalog.pull_secret.vault.name}' with content " + f"'{custom_catalog.pull_secret.vault.content}' is MANDATORY when using custom catalog. " + f"Please ensure the vault is properly configured and accessible." + ) + + current_secret = _current_pull_secret_json() + required_registries = (*RHOAI_CATALOG_REGISTRIES, *RHOAI_REGISTRIES) + + if _registries_present(current_secret, required_registries): + logger.info( + "RHOAI registries already present in cluster pull secret: %s", + ", ".join(required_registries), + ) + return + + logger.info( + "Adding %s to cluster pull secret from %s", + ", ".join(required_registries), + pull_secret_path, + ) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as temp_file: + temp_file.write(json.dumps(current_secret, indent=2, sort_keys=True)) + temp_path = Path(temp_file.name) + + try: + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=pull_secret_path, + registries=RHOAI_CATALOG_REGISTRIES, + temp_path=temp_path, + ) + + if custom_catalog.staging_pull_secret is not None: + staging_pull_secret_path = vault.get_vault_content_path( + custom_catalog.staging_pull_secret.vault.name, + custom_catalog.staging_pull_secret.vault.content, + ) + if staging_pull_secret_path is None: + raise RuntimeError( + f"RHOAI custom catalog is enabled with staging pull secret configured, but the " + f"required staging pull secret is not available. " + f"Vault '{custom_catalog.staging_pull_secret.vault.name}' with content " + f"'{custom_catalog.staging_pull_secret.vault.content}' is MANDATORY when staging pull secret is configured. " + f"Please ensure the vault is properly configured and accessible." + ) + + current_secret = _apply_vault_pull_secret( + current_secret=current_secret, + pull_secret_path=staging_pull_secret_path, + registries=RHOAI_REGISTRIES, + temp_path=temp_path, + ) + oc( + "set", + "data", + f"secret/{RHOAI_PULL_SECRET_NAME}", + "-n", + RHOAI_PULL_SECRET_NAMESPACE, + f"--from-file=.dockerconfigjson={temp_path}", + ) + finally: + temp_path.unlink(missing_ok=True) + + wait_for_rhoai_pull_secret_ready() + + +def ensure_required_crds_before_dsc(rhoai: RhoaiOperatorConfig) -> None: + wait_for_crds_command.run( + crd_names=rhoai.required_crds_before_dsc, + display_name="RHOAI pre-DSC CRDs", + ) + + +def prepare_rhoai_operator( + *, + platform: dict[str, Any], + rhoai: dict[str, Any], + icsp_applier: Callable[[], None], +) -> None: + rhoai_config = RhoaiOperatorConfig.model_validate(rhoai) + + prepare_rhcl_operator(platform) + if rhoai_config.custom_catalog.enabled: + prepare_rhoai_pull_secret(rhoai_config.custom_catalog) + icsp_applier() + deploy_rhoai_custom_catalog(custom_catalog=rhoai_config.custom_catalog) + operator_spec = operator_spec_by_package(platform, "rhods-operator") + operator_spec = rhoai_operator_spec( + custom_catalog=rhoai_config.custom_catalog, + operator_spec=operator_spec, + ) + ensure_operator_subscription(operator_spec) + ensure_required_crds_before_dsc(rhoai_config) diff --git a/projects/rhoai/library/pr_args.py b/projects/rhoai/library/pr_args.py new file mode 100644 index 000000000..6d10dccd6 --- /dev/null +++ b/projects/rhoai/library/pr_args.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +RHOAI PR directive handlers. + +Provides directive handlers for RHOAI-specific PR trigger directives. +These handlers are consumed by project-level pr_args modules (e.g., llm_d). + +Supported syntax:: + + /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 +""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + + +def handle_rhoai_rc_image_directive(line: str) -> dict[str, Any]: + """ + Handle /rhoai.rc-image directive for setting RHOAI RC catalog image. + + Format: /rhoai.rc-image IMAGE + + Args: + line: The directive line + + Returns: + Dictionary with custom catalog and operator channel configuration + + Raises: + ValueError: If image is empty + """ + image = line.removeprefix("/rhoai.rc-image").strip() + + if not image: + raise ValueError( + "Invalid /rhoai.rc-image directive: image cannot be empty. " + "Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123" + ) + + return { + "platform.rhoai.custom_catalog.enabled": True, + "platform.rhoai.custom_catalog.image": image, + "platform.operators.rhods-operator.channel": "beta", + } + + +def get_rhoai_directive_handlers() -> dict[str, callable]: + """ + Get a mapping of RHOAI directive prefixes to their handler functions. + + Returns: + Dictionary mapping directive prefixes to handler functions + """ + return { + "/rhoai.rc-image": handle_rhoai_rc_image_directive, + } + + +def get_supported_rhoai_directives() -> dict[str, str]: + """ + Get a dictionary of supported RHOAI directives and their descriptions. + + Returns: + Dictionary mapping directive names to detailed descriptions + """ + return { + "/rhoai.rc-image": """Set RHOAI RC catalog image for testing release candidates. + Format: /rhoai.rc-image IMAGE + Example: /rhoai.rc-image quay.io/rhoai/rhoai-fbc-fragment@sha256:abc123 + Effect: Enables custom catalog with the given image and sets operator channel to beta.""", + } diff --git a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py index 27a1d1631..9053068b0 100644 --- a/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py +++ b/projects/rhoai/toolbox/wait_datasciencecluster_ready/main.py @@ -61,30 +61,64 @@ def capture_initial_dsc(args, ctx): def wait_for_datasciencecluster_ready(args, ctx): """Wait for the DataScienceCluster phase to become Ready""" - # Query only the status.phase field and show output + # Use plain text output to show both READY and REASON columns result = oc( "get", "datasciencecluster", args.datasciencecluster_name, "-n", args.namespace, - "-o", - "jsonpath={.status.phase}", - log_stdout=True, # Show the output + log_stdout=True, # Show the table output check=False, ) - phase = result.stdout.strip() if result.stdout else None - - if phase == "Ready": - return "DataScienceCluster ready" - if phase in {"Failed", "Error"}: - raise RuntimeError(f"DataScienceCluster entered terminal phase {phase}") - # Provide specific reason for retry - if not phase: - return (False, "DataScienceCluster status.phase is empty, retrying...") + # Abort if the DataScienceCluster is not found + if result.returncode != 0: + error_msg = f"DataScienceCluster {args.datasciencecluster_name} not found in namespace {args.namespace}" + if result.stderr: + error_msg += f": {result.stderr.strip()}" + raise RuntimeError(error_msg) + + # Parse the table output into a dict + lines = result.stdout.strip().split("\n") + if len(lines) < 2: + parsed_data = {} else: - return (False, f"DataScienceCluster is in {phase} phase, waiting for Ready...") + header = lines[0].split() + data = lines[1].split() + # Handle case where REASON column might be empty + while len(data) < len(header): + data.append("") + parsed_data = dict(zip(header, data, strict=True)) + + # Check if we have the required columns + if not parsed_data: + return (False, "Unable to parse DataScienceCluster output, waiting...") + + ready_value = parsed_data.get("READY", "").strip() + reason_value = parsed_data.get("REASON", "").strip() + + # Check if READY column shows True + if ready_value.lower() == "true": + return f"DataScienceCluster ready (READY={ready_value})" + + # Fail fast if READY shows False and REASON shows Error + if ready_value.lower() == "false" and "error" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster is in Error state (READY={ready_value}, REASON={reason_value}). Check the DSC status and logs for details." + ) + + # Fail fast if REASON shows Failed + if "failed" in reason_value.lower(): + raise RuntimeError( + f"DataScienceCluster failed (REASON={reason_value}). Check the DSC status and logs for details." + ) + + # Still waiting - provide context for retry + return ( + False, + f"DataScienceCluster is not ready yet (READY={ready_value}, REASON={reason_value or 'N/A'}), waiting...", + ) @always diff --git a/projects/skeleton/README.md b/projects/skeleton/README.md index d496dba30..7595f1f38 100644 --- a/projects/skeleton/README.md +++ b/projects/skeleton/README.md @@ -277,17 +277,11 @@ def prepare(self): self.log("Starting prepare phase...") # Example: Install dependencies - if not self.execute_command( - "oc apply -f manifests/setup.yaml", - "Deploy setup resources" - ): + if not self.execute_command("oc apply -f manifests/setup.yaml", "Deploy setup resources"): return 1 # Example: Validate environment - if not self.execute_command( - "oc get nodes", - "Check cluster nodes" - ): + if not self.execute_command("oc get nodes", "Check cluster nodes"): return 1 self.log("Prepare phase completed!", "success") @@ -301,16 +295,12 @@ def test(self): # Example: Run performance tests if not self.execute_command( - "python scripts/performance_test.py --config config.yaml", - "Running performance tests" + "python scripts/performance_test.py --config config.yaml", "Running performance tests" ): return 1 # Example: Run functional tests - if not self.execute_command( - "pytest tests/ -v", - "Running functional tests" - ): + if not self.execute_command("pytest tests/ -v", "Running functional tests"): return 1 self.log("Test phase completed!", "success") @@ -323,16 +313,10 @@ def cleanup(self): self.log("Starting cleanup phase...") # Example: Remove test resources - self.execute_command( - "oc delete -f manifests/", - "Cleanup test resources" - ) + self.execute_command("oc delete -f manifests/", "Cleanup test resources") # Example: Generate reports - self.execute_command( - "python scripts/generate_report.py", - "Generate final report" - ) + self.execute_command("python scripts/generate_report.py", "Generate final report") self.log("Cleanup phase completed!", "success") return 0 @@ -358,10 +342,7 @@ if not success: return 1 # Exit with error # Command with complex logic -result = self.execute_command( - "kubectl get pods -o json", - "Check pod status" -) +result = self.execute_command("kubectl get pods -o json", "Check pod status") ``` ### 3. Error Handling @@ -383,10 +364,10 @@ self.execute_command("cleanup-command", "Optional cleanup") Use the logging methods for consistent output: ```python -self.log("Starting operation", "info") # ℹ️ [project] Starting operation +self.log("Starting operation", "info") # ℹ️ [project] Starting operation self.log("Operation completed", "success") # ✅ [project] Operation completed -self.log("Warning occurred", "warning") # ⚠️ [project] Warning occurred -self.log("Error occurred", "error") # ❌ [project] Error occurred +self.log("Warning occurred", "warning") # ⚠️ [project] Warning occurred +self.log("Error occurred", "error") # ❌ [project] Error occurred ``` ### 5. Verbose Mode @@ -407,7 +388,7 @@ The skeleton uses Click groups to organize commands: ```python @click.group() -@click.option('--verbose', '-v', is_flag=True, help='Enable verbose output') +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") @click.pass_context def cli(ctx, verbose): """Project CI Operations for FORGE.""" @@ -415,6 +396,7 @@ def cli(ctx, verbose): ctx.obj.verbose = verbose ctx.obj.runner = YourProjectTestRunner(verbose) + @cli.command() @click.pass_context def prepare(ctx): diff --git a/projects/skeleton/orchestration/ci.py b/projects/skeleton/orchestration/ci.py index 6dfef28ac..d8efa8e5c 100755 --- a/projects/skeleton/orchestration/ci.py +++ b/projects/skeleton/orchestration/ci.py @@ -102,6 +102,17 @@ def preflight(ctx) -> int: return 0 +@main.command() +@click.pass_context +@ci_lib.safe_ci_entrypoint +def launcher(ctx) -> int: + """Launcher job - Deploys fournos jobs.""" + + logger.warning("Nothing so far for the launcher job") + + return 0 + + main.add_command(caliper_export_entrypoint) main.add_command(caliper_replot_entrypoint) diff --git a/projects/skeleton/orchestration/config.yaml b/projects/skeleton/orchestration/config.yaml index 4c15f6591..d0662ae62 100644 --- a/projects/skeleton/orchestration/config.yaml +++ b/projects/skeleton/orchestration/config.yaml @@ -60,7 +60,7 @@ caliper: no_cache: false visualize: enabled: true - output_dir: null + output_dir: reports reports: [summary_table, throughput_chart] report_group: null @@ -77,16 +77,10 @@ caliper: enabled: true output_dir: ai_data - analyze: - enabled: true - historical_kpis: historical_data - output: regression_analyze/kpi_analyze.json - s3: bucket: psap-dashboard-data instance: smoke directory: skeleton-dashboard - prefix: skeleton/ vault: name: psap-forge-aws-s3-export aws_credentials_file: aws.credentials @@ -105,8 +99,15 @@ caliper: include_kpis_json: true include_ai_data: true + analyze: + enabled: true + current_kpis: kpis/kpis.json + historical_kpis: historical_data + output: regression_analyze/kpi_analyze.json + + agentic: - enabled: true + enabled: false model_key: qwen-3-6-35b on_failure: enabled: true diff --git a/projects/skeleton/orchestration/presets.d/presets.yaml b/projects/skeleton/orchestration/presets.d/presets.yaml index e49eca21f..211f236a8 100644 --- a/projects/skeleton/orchestration/presets.d/presets.yaml +++ b/projects/skeleton/orchestration/presets.d/presets.yaml @@ -16,3 +16,6 @@ medium_test: long_test: skeleton.test.duration_seconds: 300 # Long 5 minute test + +agent: + agentic.enabled: true diff --git a/projects/skeleton/postprocess/default/plotting/summary_table.py b/projects/skeleton/postprocess/default/plotting/summary_table.py index 8e19ef2a7..e758b5e03 100644 --- a/projects/skeleton/postprocess/default/plotting/summary_table.py +++ b/projects/skeleton/postprocess/default/plotting/summary_table.py @@ -3,10 +3,13 @@ from __future__ import annotations import html as html_lib +import logging from pathlib import Path from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class SummaryTablePlot: """Generates an HTML summary table of test results.""" @@ -23,11 +26,19 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating summary table HTML report") + logger.info(f"Processing {len(model.unified_result_records)} result records") + rows = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): scenario = r.distinguishing_labels.get("scenario", "") tp = r.metrics.get("throughput", "") lat = r.metrics.get("latency_ms", "") + + logger.debug( + f"Row {i}: path={r.test_base_path}, scenario={scenario}, throughput={tp}, latency={lat}" + ) + rows.append( "" f"{html_lib.escape(str(r.test_base_path))}" @@ -37,6 +48,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: "" ) + logger.info(f"Generated {len(rows)} table rows") + table_html = ( "" "Skeleton sample — summary" @@ -50,5 +63,8 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: ) output_file = output_dir / "summary_table.html" + logger.info(f"Writing summary table to: {output_file}") output_file.write_text(table_html, encoding="utf-8") + file_size = output_file.stat().st_size + logger.info(f"Summary table HTML written successfully ({file_size} bytes)") return str(output_file) diff --git a/projects/skeleton/postprocess/default/plotting/throughput_chart.py b/projects/skeleton/postprocess/default/plotting/throughput_chart.py index 075ccd610..774d92f84 100644 --- a/projects/skeleton/postprocess/default/plotting/throughput_chart.py +++ b/projects/skeleton/postprocess/default/plotting/throughput_chart.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging from pathlib import Path import plotly.graph_objects as go from projects.caliper.engine.model import UnifiedRunModel +logger = logging.getLogger(__name__) + class ThroughputChartPlot: """Generates a Plotly bar chart of throughput by scenario.""" @@ -24,21 +27,34 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: Returns: Path to the generated HTML file """ + logger.info("Generating throughput chart HTML report") + logger.info( + f"Processing {len(model.unified_result_records)} result records for throughput data" + ) + xs: list[str] = [] ys: list[float] = [] - for r in model.unified_result_records: + for i, r in enumerate(model.unified_result_records): label = str(r.distinguishing_labels.get("scenario") or r.test_base_path) raw = r.metrics.get("throughput", 0) try: y = float(raw) except (TypeError, ValueError): + logger.debug( + f"Record {i}: Could not convert throughput '{raw}' to float, using 0.0" + ) y = 0.0 + logger.debug(f"Record {i}: label='{label}', throughput={y}") xs.append(label) ys.append(y) + logger.info(f"Collected throughput data for {len(xs)} scenarios") + logger.info(f"Throughput range: {min(ys) if ys else 0} - {max(ys) if ys else 0}") + + logger.info("Creating Plotly bar chart") fig = go.Figure(data=[go.Bar(x=xs, y=ys)]) fig.update_layout( title="Throughput by scenario", @@ -49,5 +65,12 @@ def generate(model: UnifiedRunModel, output_dir: Path) -> str: from projects.caliper.postprocess.helpers.visualization_utils import write_full_page_html output_file = output_dir / "throughput_chart.html" - write_full_page_html(fig, str(output_file), "Throughput Chart") - return str(output_file) + logger.info(f"Writing throughput chart to: {output_file}") + success = write_full_page_html(fig, str(output_file), "Throughput Chart") + if success: + file_size = output_file.stat().st_size + logger.info(f"Throughput chart HTML written successfully ({file_size} bytes)") + return str(output_file) + else: + logger.error(f"Failed to write throughput chart to {output_file}") + return None diff --git a/projects/skeleton/postprocess/default/plugin.py b/projects/skeleton/postprocess/default/plugin.py index 14fe301b2..18b00075f 100644 --- a/projects/skeleton/postprocess/default/plugin.py +++ b/projects/skeleton/postprocess/default/plugin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any @@ -15,6 +16,8 @@ from .parsing import SkeletonKpiHandler, SkeletonParser from .plotting import SummaryTablePlot, ThroughputChartPlot +logger = logging.getLogger(__name__) + class SkeletonDefaultPlugin(PostProcessingPlugin): """ @@ -47,16 +50,42 @@ def visualize( visualize_config: dict[str, Any] | None, ) -> list[str]: """Generate visualization reports.""" + logger.info("Starting skeleton plugin visualization") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Requested report IDs: {report_ids}") + logger.info(f"Group ID: {group_id}") + logger.info(f"Available plot types: {list(self.plots.keys())}") + logger.info(f"Model contains {len(model.unified_result_records)} result records") + output_dir.mkdir(parents=True, exist_ok=True) paths: list[str] = [] wanted = frozenset(report_ids or ()) + invalid_plots: list[str] = [] + + logger.info(f"Will generate plots for: {sorted(wanted)}") for report_id in wanted: if report_id in self.plots: + logger.info(f"Generating plot: {report_id}") plot_class = self.plots[report_id] path = plot_class.generate(model, output_dir) + logger.info(f"Generated plot file: {path}") paths.append(path) - + else: + logger.warning(f"Requested plot type '{report_id}' not available in this plugin") + invalid_plots.append(report_id) + + # Check for invalid plot types and raise error if found + if invalid_plots: + available_types = list(self.plots.keys()) + raise ValueError( + f"Invalid plot types requested: {sorted(invalid_plots)}. " + f"Available types in this plugin: {sorted(available_types)}" + ) + + logger.info( + f"Skeleton plugin visualization completed. Generated {len(paths)} files: {paths}" + ) return paths def kpi_catalog(self) -> list[dict[str, Any]]: diff --git a/pyproject.toml b/pyproject.toml index 400f0eb8f..4faa72b56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dev = [ "pytest>=7.4.0", "pytest-asyncio>=0.21.1", "black>=23.7.0", - "ruff>=0.0.287", + "ruff==0.16.0", "pre-commit>=4.0", ] docs = [ diff --git a/vaults/psap-forge-dashboard-s3.yaml b/vaults/psap-forge-dashboard-s3.yaml new file mode 100644 index 000000000..ada9243ed --- /dev/null +++ b/vaults/psap-forge-dashboard-s3.yaml @@ -0,0 +1,15 @@ +env_key: PSAP_FORGE_DASHBOARD_S3_SECRET_PATH +description: | + AWS credentials for dashboard CSV sync and data uploads + to S3 buckets used by Forge projects. + +content: + aws.credentials: + description: | + AWS credentials file (INI format). The key must have + s3:GetObject, s3:PutObject, s3:CopyObject on: + - psap-dashboard-data (CSV dashboard data, profiler traces) + sample: | + [default] + aws_access_key_id = AKIA... + aws_secret_access_key = ..... diff --git a/vaults/psap-forge-staging-image-pull.yaml b/vaults/psap-forge-staging-image-pull.yaml new file mode 100644 index 000000000..5979f3b4d --- /dev/null +++ b/vaults/psap-forge-staging-image-pull.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_FORGE_STAGING_IMAGE_PULL_SECRET_PATH +description: | + Vault containing registry credentials for staging image pulling + +content: + .dockerconfigjson: + description: | + Docker registry auth config used to extend the cluster pull secret + sample: | + {"auths":{"registry.stage.redhat.io":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii":{"auth":"base64-encoded-credentials"},"registry.stage.redhat.io/rhaii-early-access":{"auth":"base64-encoded-credentials"}}} diff --git a/vaults/psap-rhoai-rc.yaml b/vaults/psap-rhoai-rc.yaml new file mode 100644 index 000000000..e35cd4560 --- /dev/null +++ b/vaults/psap-rhoai-rc.yaml @@ -0,0 +1,10 @@ +env_key: PSAP_RHOAI_RC_SECRET_PATH +description: | + Vault containing credentials for the RHOAI release-candidate registry + +content: + rhoai_rc.secret: + description: | + Basic auth string used to extend the cluster pull secret for quay.io/rhoai + sample: | + rhoai+rhoai_external_readonly_bot:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx