solidify the metrics to mlflow

.claude/settings.json

@@ -0,0 +1,36 @@
+{
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Write",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"",
+            "statusMessage": "Formatting Python code with ruff..."
+          },
+          {
+            "type": "command",
+            "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"",
+            "statusMessage": "Fixing Python linting issues with ruff..."
+          }
+        ]
+      },
+      {
+        "matcher": "Edit",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff format . || echo \"Warning: ruff format failed for $f\"; }; } || echo \"Warning: file path processing failed\"",
+            "statusMessage": "Formatting Python code with ruff..."
+          },
+          {
+            "type": "command",
+            "command": "jq -r '.tool_input.file_path' | { read -r f; [[ \"$f\" == *.py ]] && { ruff check --fix . || echo \"Warning: ruff check --fix failed for $f\"; }; } || echo \"Warning: file path processing failed\"",
+            "statusMessage": "Fixing Python linting issues with ruff..."
+          }
+        ]
+      }
+    ]
+  }
+}

.github/steps/prepare_minimal_ubuntu_image/action.yml

@@ -1,4 +1,5 @@
 name: "Prepare FORGE minimal ubuntu image"
+description: "Install minimal Python dependencies required for FORGE repository operations"
 
 runs:
   using: "composite" # <-- this is the important part

.github/workflows/check_consistency.yml

@@ -18,11 +18,31 @@ jobs:
       PYTHONPATH: ${{ github.workspace }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - uses: ./.github/steps/prepare_minimal_ubuntu_image
 
       - name: Check that no symlink is broken
         run: bin/run_ansible_toolbox.py repo validate_no_broken_link
+
+      - name: Check that all source files end with EOL
+        run: |
+          # Find all source files and check they end with newline
+          files_without_eol=()
+
+          while IFS= read -r -d '' file; do
+            if [[ -s "$file" ]] && [[ -n "$(tail -c1 "$file")" ]]; then
+              files_without_eol+=("$file")
+            fi
+          done < <(find . -path './.git' -prune -o \( -name "*.py" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" -o -name "*.md" \) -type f -print0)
+
+          if [[ ${#files_without_eol[@]} -gt 0 ]]; then
+            echo "❌ Files missing final newline:"
+            printf '%s\n' "${files_without_eol[@]}"
+            echo "Fix with: echo >> filename"
+            exit 1
+          fi
+          echo "✅ All source files end with newline"

.github/workflows/check_python_syntax.yml

@@ -10,9 +10,10 @@ jobs:
       max-parallel: 4
     steps:
       - name: Use checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Check that all the files can be compiled
         run: |

.github/workflows/image-push.yaml

@@ -10,7 +10,9 @@ jobs:
   image-push:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
 
       - name: Remove GitHub credentials from git config
         run: |

.github/workflows/ruff.yaml

@@ -16,7 +16,9 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: false
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5

.github/workflows/test_toolbox_dsl.yml

@@ -20,7 +20,9 @@ jobs:
       PYTHONPATH: ${{ github.workspace }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
 
       - uses: actions/setup-python@v5
         with:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.16.0
     hooks:
       - id: ruff
         args: [--fix]

AGENTS.md

@@ -0,0 +1,89 @@
+# FORGE CI Framework — Agent Instructions
+
+## Project Overview
+
+FORGE is a CI/CD testing framework that orchestrates Kubernetes workloads on OpenShift clusters.
+It uses a Python DSL to define tasks, apply manifests, and collect artifacts for debugging.
+
+Key paths:
+- `projects/` — individual project test harnesses
+- `projects/NAME/orchestration` — upper layer of the test harness (config, prepare, test, cleanup)
+- `projects/NAME/toolbox` — lower layer of the test harness (toolbox of scripts altering the state of the cluster, using the DSL language)
+- `projects/NAME/test` — unit testing of the test harness
+- `projects/core/dsl/` — shared DSL (task execution, k8s utilities, shell helpers)
+- `fournos/gitops/` — Tekton pipelines and GitOps base workflows
+- `docs/` — project documentation
+
+## Security: Secret Handling (CRITICAL)
+
+**NEVER write secrets or sensitive data to artifact directories, logs, or files.**
+
+### Rules
+
+1. **Never persist secrets to `ARTIFACT_DIR`.**
+   The `oc_apply(path, manifest)` helper writes the manifest YAML to disk before applying it.
+   If the manifest contains sensitive data (passwords, tokens, pull-secrets, certificates, API keys),
+   it MUST NOT be written to any path under `env.ARTIFACT_DIR`.
+
+2. **Use `oc("apply", "-f", "-", input_text=..., handled_secretly=True)` for secrets.**
+   Apply secrets via stdin or a dedicated helper that does not persist the payload.
+   Never use the standard `oc_apply()` for Secret-kind manifests.
+
+3. **Never log secret values.**
+   Do not print, log, or include in error messages: passwords, tokens, bearer credentials,
+   TLS keys, pull-secret `.dockerconfigjson`, or any `data:`/`stringData:` field from a Secret.
+
+4. **Check the `kind:` field.**
+   Before calling `oc_apply(artifact_path, manifest)`, verify that `manifest.get("kind") != "Secret"`.
+   If it is a Secret, use a non-persisting apply path.
+
+5. **Artifact directories are public.**
+   Treat everything under `ARTIFACT_DIR` as if it will be published on the internet.
+   Only write non-sensitive operational data (CRs, logs, pod descriptions, events).
+   Pod/Deployment and other resources are safe to store, as they must not contain secrets. See point 9.
+   Kubernetes Secret objects should never be stored in the artifacts. Kubernetes Secret descriptions can be safely stored.
+
+7. **Use the vault module.**
+    Use the project `projects.core.library.vault` module to access the secrets, stored in files. Consider all the content of these files as secret.
+
+8. **Never pass secrets via environment variables**
+    For transparency and post-mortem troubleshooting, the environment variables are saved to disk at multiple locations. Do not use environment variables to pass a secret between two components. There might be few exceptions to this rule (MLFlow, AWS), but they must be handled with care.
+
+9. **Never pass secrets via Pod environment variables**
+   The Pod definition will be stored in public artifacts. Do no use plain text Pod environment variables to pass secrets to the Pod. Instead, use Secret resources and Pod environment variables that reference this secret.
+
+### Safe Patterns
+
+```python
+# GOOD: Apply a Secret without persisting to disk
+def apply_secret(manifest: dict) -> None:
+    """Apply a Secret manifest via stdin — nothing written to disk."""
+    import yaml
+    from projects.core.dsl.utils.k8s import oc
+
+    oc("apply", "-f", "-", input_text=yaml.safe_dump(manifest, sort_keys=False))
+
+
+# GOOD: Guard before using oc_apply
+def safe_oc_apply(artifact_path, manifest):
+    if manifest.get("kind") == "Secret":
+        raise ValueError(
+            f"Refusing to write Secret '{manifest['metadata']['name']}' to artifacts. "
+            "Use apply_secret() instead."
+        )
+    oc_apply(artifact_path, manifest)
+```
+
+### Forbidden Patterns
+
+```python
+# BAD: Writing a pull-secret to artifacts
+src_dir = env.ARTIFACT_DIR / "src"
+oc_apply(src_dir / "image-pull-secret.yaml", secret_manifest)  # LEAKS CREDENTIALS
+
+# BAD: Logging secret content
+logger.info(f"Applying secret: {manifest}")  # LEAKS if manifest has sensitive data
+
+# BAD: Including tokens in error messages
+raise RuntimeError(f"Failed to auth with token {token}")  # LEAKS TOKEN
+```

OWNERS

@@ -6,10 +6,12 @@ approvers:
   - sjmonson
   - ashtarkb
   - MML-coder
+  - Harshith-umesh
 reviewers:
   - ashishkamra
   - albertoperdomo2
   - tosokin
   - sjmonson
   - ashtarkb
   - MML-coder
+  - Harshith-umesh

docs/caliper/test-labels-format.md

@@ -0,0 +1,130 @@
+# Test Labels Format (`__test_labels__.yaml`)
+
+## Purpose
+
+The `__test_labels__.yaml` file marks directories as Caliper test bases and provides metadata for test identification, grouping, and status tracking.
+
+## File Format
+
+```yaml
+version: "1"
+labels:
+  # Test distinguishing characteristics  
+  key1: value1
+  key2: value2
+kpi_labels:
+  # System/environment context for KPI labeling
+  platform: "CKS"
+  gpu_type: "H100"
+  test_harness: "guidellm"
+completion:
+  # Test execution status (added at completion)
+  success: true|false
+  message: "Status description"
+```
+
+### Required Fields
+- **`version`**: Schema version (currently `"1"`)
+- **`labels`**: Key-value pairs describing test characteristics
+
+### Optional Fields
+- **`kpi_labels`**: System/environment context labels for KPI records
+  - **`platform`**: Platform name (e.g., `"CKS"`, `"RHOAI"`)
+  - **`gpu_type`**: GPU type (e.g., `"H100"`, `"A100"`)
+  - **`test_harness`**: Test framework (e.g., `"guidellm"`, `"vllm"`)
+- **`completion`**: Test execution status
+  - **`success`**: `true` if succeeded, `false` if failed
+  - **`message`**: Human-readable status description
+
+## Labels vs KPI Labels
+
+### Labels (Test Configuration)
+- **Purpose**: Distinguish different test configurations and variations
+- **Usage**: Used for test filtering, grouping, and analysis
+- **Examples**: `model_name`, `deployment_profile`, `guidellm_loadshape`
+- **Scope**: Specific to the test configuration
+
+### KPI Labels (System Context)
+- **Purpose**: Provide system/environment context for KPI records
+- **Usage**: Applied to all KPI metrics from the test, enables cross-environment analysis
+- **Examples**: `platform`, `gpu_type`, `test_harness`
+- **Scope**: Describes the execution environment
+
+## Common Labels
+
+- **`model_name`**: AI/ML model name (e.g., `"llama-3.1-8b"`)
+- **`deployment_profile`**: Deployment variant (e.g., `"simple-tp2-x4"`)  
+- **`guidellm_loadshape`**: Benchmark load pattern (e.g., `"heavy-heterogeneous"`)
+- **`benchmark_type`**: Benchmark type (e.g., `"throughput"`, `"latency"`)
+
+## Common KPI Labels
+
+- **`platform`**: Target platform (e.g., `"CKS"`, `"RHOAI"`)
+- **`gpu_type`**: GPU hardware type (e.g., `"H100"`, `"A100"`)
+- **`test_harness`**: Testing framework (e.g., `"guidellm"`, `"vllm"`)
+
+## Examples
+
+### Basic Test
+```yaml
+version: "1"
+labels:
+  model_name: "llama-3.1-8b"
+  deployment_profile: "simple"
+  guidellm_loadshape: "default"
+kpi_labels:
+  platform: "CKS"
+  gpu_type: "H100"
+  test_harness: "guidellm"
+```
+
+### Completed Test
+```yaml
+version: "1"
+labels:
+  model_name: "llama-3.1-8b"
+  deployment_profile: "simple-tp2-x4"
+  guidellm_loadshape: "heavy-heterogeneous"
+kpi_labels:
+  platform: "CKS"
+  gpu_type: "H100"
+  test_harness: "guidellm"
+completion:
+  success: true
+  message: "Test completed successfully"
+```
+
+### Failed Test
+```yaml
+version: "1"
+labels:
+  model_name: "gpt-4o"
+  deployment_profile: "distributed"
+kpi_labels:
+  platform: "RHOAI"
+  gpu_type: "A100"
+  test_harness: "vllm"
+completion:
+  success: false
+  message: "Connection timeout to inference service"
+```
+
+## Usage
+
+- **Test Discovery**: Caliper scans for `__test_labels__.yaml` files to find test results
+- **Test Analysis**: Labels enable filtering and comparative analysis of test configurations
+- **KPI Labeling**: KPI labels are automatically applied to all generated KPI metrics
+- **Cross-Environment Analysis**: KPI labels enable performance comparisons across different platforms/hardware
+- **Report Generation**: Both labels and KPI labels provide context for charts and report organization
+- **Status Tracking**: Completion field tracks test execution outcome
+
+## Best Practices
+
+- **Labels**: Use for test-specific configuration and distinguishing characteristics
+- **KPI Labels**: Use for system/environment context that applies to all KPIs
+- Generate both label sets early, add completion status at test end
+- Use consistent, descriptive keys across related tests
+- Use lowercase with underscores (e.g., `model_name`, `gpu_type`)
+- Keep label schemas stable within projects
+- Populate KPI labels from configuration to ensure consistency (e.g., from `cpt.kpi.labels.*`)
+- Missing KPI labels should be handled gracefully with fallback values