diff --git a/hacks/sync_vault_secrets.py b/hacks/sync_vault_secrets.py deleted file mode 100755 index 99bf354..0000000 --- a/hacks/sync_vault_secrets.py +++ /dev/null @@ -1,454 +0,0 @@ -#!/usr/bin/env python3 -"""Synchronize secrets from a HashiCorp Vault to Kubernetes. - -Reads entries from a Vault KV v2 engine and creates one Kubernetes -Opaque Secret per vault entry in the dedicated secrets namespace. -Existing secrets are updated in-place. - -Required environment variables ------------------------------- -* ``VAULT_TOKEN`` — Vault authentication token - -Optional environment variables ------------------------------- -* ``VAULT_ADDR`` — Vault server URL (default: ``https://vault.ci.openshift.org``) -* ``VAULT_SECRET_PATH`` — path within the KV engine (default: ``selfservice/psap``) -* ``VAULT_KV_MOUNT`` — KV v2 engine mount point (default: ``kv``) -* ``FOURNOS_SECRETS_NAMESPACE`` — target Kubernetes namespace (can also - use ``-n``). Defaults to ``psap-secrets``. - -Prerequisites -------------- -* ``kubectl`` / ``oc`` context pointing at the target cluster, or - in-cluster service-account credentials. - -Examples --------- -Sync all vault entries under the configured path (using all defaults):: - - export VAULT_TOKEN="s.xxxxx" - python hacks/sync_vault_secrets.py - -Use custom vault, path, and namespace:: - - export VAULT_TOKEN="s.xxxxx" - export VAULT_ADDR="https://vault.example.com" - export VAULT_SECRET_PATH="path/to/secrets" - python hacks/sync_vault_secrets.py -n my-namespace - -Preview without touching the cluster:: - - python hacks/sync_vault_secrets.py --dry-run -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import re -import urllib.error -import urllib.request - -# Set FOURNOS_WORKLOAD_NAMESPACE if not set (required for fournos imports) -if not os.environ.get("FOURNOS_WORKLOAD_NAMESPACE"): - os.environ["FOURNOS_WORKLOAD_NAMESPACE"] = "not used" - -from fournos.core.constants import LABEL_VAULT_ENTRY -from fournos.settings import settings - -logger = logging.getLogger(__name__) - -KV_MOUNT_DEFAULT = "kv" -VAULT_ADDR_DEFAULT = "https://vault.ci.openshift.org" -VAULT_SECRET_PATH_DEFAULT = "selfservice/psap" -NAMESPACE_DEFAULT = "psap-secrets" - -LABEL_MANAGED_BY = "app.kubernetes.io/managed-by" -ANNOTATION_VAULT_ADDR = "fournos.dev/vault-addr" -ANNOTATION_VAULT_PATH = "fournos.dev/vault-path" -MANAGER_VALUE = "fournos-vault-sync" -SKIP_SYNC_KEY = ( - "fournos_skip_sync" # vaults with this key won't be synchronized to the cluster -) - - -# --------------------------------------------------------------------------- -# Vault helpers -# --------------------------------------------------------------------------- - - -def _vault_request( - addr: str, - path: str, - token: str, - method: str = "GET", -) -> dict: - url = f"{addr}/v1/{path}" - req = urllib.request.Request(url, method=method) - req.add_header("X-Vault-Token", token) - logger.debug("%s %s", method, url) - try: - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) - except urllib.error.HTTPError as exc: - raise RuntimeError( - f"Vault {method} {url} failed (HTTP {exc.code}): {exc.reason}" - ) from exc - except urllib.error.URLError as exc: - raise RuntimeError( - f"Vault {method} {url} failed (connection error): {exc.reason}" - ) from exc - - -def vault_list( - addr: str, - mount: str, - path: str, - token: str, -) -> list[str]: - """List entries under a KV v2 path.""" - api_path = f"{mount}/metadata/{path}" - data = _vault_request(addr, api_path, token, method="LIST") - return data.get("data", {}).get("keys", []) - - -def vault_read( - addr: str, - mount: str, - path: str, - token: str, -) -> dict[str, str]: - """Read key-value pairs from a single KV v2 secret.""" - api_path = f"{mount}/data/{path}" - data = _vault_request(addr, api_path, token) - return data.get("data", {}).get("data", {}) - - -# --------------------------------------------------------------------------- -# Kubernetes helpers (lazy import so --dry-run works without kubeconfig) -# --------------------------------------------------------------------------- - - -def _k8s_core_api(): - from kubernetes import client - from kubernetes import config as k8s_config - - try: - k8s_config.load_kube_config() - except k8s_config.ConfigException: - k8s_config.load_incluster_config() - return client.CoreV1Api() - - -def _apply_secret( - v1, - name: str, - namespace: str, - data: dict[str, str], - vault_addr: str, - vault_full_path: str, -): - """Create or update an Opaque Secret.""" - from kubernetes.client import V1ObjectMeta, V1Secret - from kubernetes.client.exceptions import ApiException - - secret = V1Secret( - api_version="v1", - kind="Secret", - metadata=V1ObjectMeta( - name=name, - namespace=namespace, - labels={ - LABEL_MANAGED_BY: MANAGER_VALUE, - LABEL_VAULT_ENTRY: "true", - }, - annotations={ - ANNOTATION_VAULT_ADDR: vault_addr, - ANNOTATION_VAULT_PATH: vault_full_path, - }, - ), - type="Opaque", - string_data=data, - ) - - try: - v1.read_namespaced_secret(name, namespace) - except ApiException as exc: - if exc.status != 404: - raise - v1.create_namespaced_secret(namespace, secret) - logger.info("Created Secret %s/%s", namespace, name) - else: - v1.replace_namespaced_secret(name, namespace, secret) - logger.info("Updated Secret %s/%s", namespace, name) - - -def _list_managed_secrets(v1, namespace: str) -> list[str]: - """List all secrets managed by fournos-vault-sync.""" - from kubernetes.client.exceptions import ApiException - - try: - label_selector = f"{LABEL_MANAGED_BY}={MANAGER_VALUE},{LABEL_VAULT_ENTRY}=true" - secrets = v1.list_namespaced_secret(namespace, label_selector=label_selector) - return [secret.metadata.name for secret in secrets.items] - except ApiException as exc: - logger.error("Failed to list managed secrets: %s", exc) - raise - - -def _delete_secret(v1, name: str, namespace: str): - """Delete a Kubernetes Secret.""" - from kubernetes.client.exceptions import ApiException - - try: - v1.delete_namespaced_secret(name, namespace) - logger.info("Deleted Secret %s/%s", namespace, name) - except ApiException as exc: - if exc.status != 404: - logger.error("Failed to delete Secret %s/%s: %s", namespace, name, exc) - raise - - -# --------------------------------------------------------------------------- -# Core logic -# --------------------------------------------------------------------------- - - -def _is_vault_metadata(key: str) -> bool: - return key.startswith("secretsync/") - - -_SECRET_KEY_RE = re.compile(r"^[-._a-zA-Z0-9]+$") - - -def _is_valid_k8s_key(key: str) -> bool: - """Return True if *key* is a valid Kubernetes Secret data key.""" - return bool(key) and len(key) <= 253 and bool(_SECRET_KEY_RE.match(key)) - - -_DNS_1123_RE = re.compile(r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$") -_DNS_1123_MAX_LEN = 253 - - -def is_valid_k8s_name(name: str) -> bool: - """Return True if *name* is a valid DNS-1123 subdomain.""" - return bool(name) and len(name) <= _DNS_1123_MAX_LEN and _DNS_1123_RE.match(name) - - -def sync( - *, - vault_addr: str, - vault_token: str, - kv_mount: str, - secret_path: str, - namespace: str, - dry_run: bool, -) -> int: - logger.info("Listing vaults under %s/%s ...", kv_mount, secret_path) - try: - entries = vault_list(vault_addr, kv_mount, secret_path, vault_token) - except RuntimeError as exc: - logger.error("Cannot reach Vault, aborting: %s", exc) - return 1 - names = [e.rstrip("/") for e in entries if not e.endswith("/")] - if not names: - logger.warning("No vault entries found under %s/%s", kv_mount, secret_path) - return 0 - - logger.info("Found %d vault entries: %s", len(names), ", ".join(names)) - - # Always create v1 client for read operations (needed for destructive sync preview) - v1 = _k8s_core_api() - - # Track which secrets we process for destructive sync - processed_secrets = set() - - # Get list of existing managed secrets for destructive sync (always needed for preview) - existing_managed_secrets = set(_list_managed_secrets(v1, namespace)) - logger.info("Found %d existing managed secrets", len(existing_managed_secrets)) - - errors = 0 - skipped_count = 0 - for vault_name in names: - full_path = f"{secret_path}/{vault_name}" - - if not is_valid_k8s_name(vault_name): - logger.error( - "Vault entry %r is not a valid DNS-1123 name, skipping", - vault_name, - ) - errors += 1 - continue - - secret_name = settings.vault_secret_pattern.format(entry=vault_name) - - try: - logger.info("Reading %s/%s ...", kv_mount, full_path) - kv_data = vault_read(vault_addr, kv_mount, full_path, vault_token) - except Exception: - logger.exception("Failed to read vault entry %s", full_path) - errors += 1 - continue - - if not kv_data: - logger.warning("Vault entry %s is empty, skipping", full_path) - continue - - # Check if vault contains fournos_skip_sync key - if so, skip this vault - if SKIP_SYNC_KEY in kv_data: - logger.info( - "Skipping vault %s (contains %s key)", vault_name, SKIP_SYNC_KEY - ) - skipped_count += 1 - processed_secrets.add(secret_name) - continue - - safe_data: dict[str, str] = {} - for k, v in kv_data.items(): - if _is_vault_metadata(k): - continue - if not _is_valid_k8s_key(k): - logger.error( - "Key %r in vault entry %s is not a valid K8s Secret key, skipping", - k, - full_path, - ) - errors += 1 - continue - safe_data[k] = v - - logger.info( - " %d key(s): %s", - len(safe_data), - ", ".join(safe_data.keys()), - ) - - if dry_run: - print(f"[dry-run] Would create/update Secret {namespace}/{secret_name}") - for key, value in safe_data.items(): - print(f" {key}: <{len(str(value))} chars>") - processed_secrets.add(secret_name) - continue - - try: - _apply_secret( - v1, - secret_name, - namespace, - safe_data, - vault_addr=vault_addr, - vault_full_path=f"{kv_mount}/{full_path}", - ) - processed_secrets.add(secret_name) - except Exception: - logger.exception( - "Failed to apply Secret %s/%s", - namespace, - secret_name, - ) - errors += 1 - - # Destructive sync: delete managed secrets that are no longer in vault - secrets_to_delete = existing_managed_secrets - processed_secrets - if secrets_to_delete: - logger.info( - "Found %d managed secrets not in vault, deleting: %s", - len(secrets_to_delete), - ", ".join(secrets_to_delete), - ) - - for secret_name in secrets_to_delete: - if dry_run: - print(f"[dry-run] Would delete Secret {namespace}/{secret_name}") - else: - try: - _delete_secret(v1, secret_name, namespace) - except Exception: - logger.exception( - "Failed to delete Secret %s/%s", namespace, secret_name - ) - errors += 1 - else: - logger.info("No managed secrets need to be deleted") - - if skipped_count > 0: - logger.info( - "Total vault entries skipped (contained %s): %d", - SKIP_SYNC_KEY, - skipped_count, - ) - - return 1 if errors else 0 - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - description="Sync secrets from a HashiCorp Vault to Kubernetes.", - epilog="Set VAULT_TOKEN in the environment. VAULT_ADDR and VAULT_SECRET_PATH have defaults.", - ) - parser.add_argument( - "-n", - "--namespace", - default=settings.secrets_namespace or NAMESPACE_DEFAULT, - help=f"Target Kubernetes namespace (default: $FOURNOS_SECRETS_NAMESPACE or {NAMESPACE_DEFAULT}).", - ) - parser.add_argument( - "--vault-addr", - default=os.environ.get("VAULT_ADDR", VAULT_ADDR_DEFAULT), - help=f"Vault server URL (default: $VAULT_ADDR or {VAULT_ADDR_DEFAULT}).", - ) - parser.add_argument( - "--kv-mount", - default=os.environ.get("VAULT_KV_MOUNT", KV_MOUNT_DEFAULT), - help=f"KV v2 engine mount point (default: {KV_MOUNT_DEFAULT}).", - ) - parser.add_argument( - "--secret-path", - default=os.environ.get("VAULT_SECRET_PATH", VAULT_SECRET_PATH_DEFAULT), - help=f"Path within the KV engine (default: $VAULT_SECRET_PATH or {VAULT_SECRET_PATH_DEFAULT}).", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be created without touching the cluster.", - ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable debug logging.", - ) - - args = parser.parse_args(argv) - - logging.basicConfig( - level=logging.DEBUG if args.verbose else logging.INFO, - format="%(levelname)-5s %(message)s", - ) - logging.getLogger("kubernetes").setLevel(logging.WARNING) - logging.getLogger("urllib3").setLevel(logging.WARNING) - - vault_token = os.environ.get("VAULT_TOKEN", "") - if not vault_token: - logger.error("VAULT_TOKEN environment variable is not set") - return 1 - - return sync( - vault_addr=args.vault_addr, - vault_token=vault_token, - kv_mount=args.kv_mount, - secret_path=args.secret_path, - namespace=args.namespace, - dry_run=args.dry_run, - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_secret_refs.py b/tests/test_secret_refs.py index a52fa89..10eac29 100644 --- a/tests/test_secret_refs.py +++ b/tests/test_secret_refs.py @@ -1,169 +1,21 @@ -"""End-to-end tests — Vault sync -> secretRef resolution -> PipelineRun. +"""End-to-end tests — secretRef resolution -> PipelineRun. SecretRefs live on the FournosJob spec and are populated by the execution -engine during the Resolving phase. The Vault HTTP layer is mocked so no real Vault is -needed, but secrets are created on the live cluster by the sync script, -then consumed by a FournosJob whose spec.secretRefs references them. +engine during the Resolving phase. -Both tests use a noop resolve Job to avoid races with the mock resolver, -and supply ``secretRefs`` directly in the FournosJob spec. +The test uses a noop resolve Job to avoid races with the mock resolver, +and supplies ``secretRefs`` directly in the FournosJob spec. """ from __future__ import annotations -import importlib.util -import pathlib -import sys -from unittest import mock - -import pytest -from kubernetes import client, config - -from fournos.core.constants import LABEL_VAULT_ENTRY from tests.conftest import ( - NAMESPACE, - SECRETS_NAMESPACE, create_job, create_noop_resolve_job, - get_pipelinerun_volumes, job_status_summary, poll_phase, ) -# --------------------------------------------------------------------------- -# Import the sync script as a module (it lives outside the package tree). -# --------------------------------------------------------------------------- - -_script = ( - pathlib.Path(__file__).resolve().parents[1] / "hacks" / "sync_vault_secrets.py" -) -_spec = importlib.util.spec_from_file_location("sync_vault_secrets", _script) -svs = importlib.util.module_from_spec(_spec) -sys.modules["sync_vault_secrets"] = svs -_spec.loader.exec_module(svs) - -# Fake Vault data: one entry named "e2e-creds" with two keys -# (plus a secretsync/ metadata key that should be filtered out). -VAULT_ENTRY = "e2e-creds" -VAULT_SECRET = f"vault-{VAULT_ENTRY}" -VAULT_DATA = { - "username": "admin", - "password": "s3cret", - "secretsync/target-namespace": "should-be-filtered", -} - - -@pytest.fixture(scope="session") -def core_v1(): - try: - config.load_incluster_config() - except config.ConfigException: - config.load_kube_config() - return client.CoreV1Api() - - -def _delete_secret_if_exists(v1, name: str, namespace: str = SECRETS_NAMESPACE) -> None: - try: - v1.delete_namespaced_secret(name, namespace) - except client.exceptions.ApiException as exc: - if exc.status != 404: - raise - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - - -def test_vault_sync_then_fjob(k8s, core_v1): - """Sync a mocked Vault entry, then verify a FournosJob passes it to PipelineRun. - - A noop resolve Job is pre-created so the mock resolver doesn't - overwrite ``secretRefs``. The FournosJob carries ``secretRefs`` - directly in its spec, avoiding any race with the resolver. - """ - - with ( - mock.patch.object(svs, "vault_list", return_value=[VAULT_ENTRY]), - mock.patch.object(svs, "vault_read", return_value=VAULT_DATA), - ): - rc = svs.sync( - vault_addr="https://vault.fake.test", - vault_token="s.fake", - kv_mount="kv", - secret_path="selfservice/e2e", - namespace=SECRETS_NAMESPACE, - dry_run=False, - ) - assert rc == 0, "sync_vault_secrets.sync() returned non-zero" - - expected_copy = f"test-e2e-secret-{VAULT_ENTRY}" - - try: - secret = core_v1.read_namespaced_secret(VAULT_SECRET, SECRETS_NAMESPACE) - assert secret.metadata.labels[LABEL_VAULT_ENTRY] == "true" - - create_noop_resolve_job("test-e2e-secret") - - create_job( - k8s, - "test-e2e-secret", - { - "cluster": "cluster-1", - "hardware": {"gpuType": "a100", "gpuCount": 2}, - "secretRefs": [VAULT_ENTRY], - "executionEngine": { - "forge": { - "project": "testproj/llmd", - "args": ["cks", "internal-test"], - } - }, - }, - ) - - phase = poll_phase( - k8s, - "test-e2e-secret", - terminal={"Running", "Succeeded", "Failed"}, - timeout=90, - ) - assert phase in ("Running", "Succeeded"), job_status_summary( - k8s, "test-e2e-secret" - ) - - volumes = get_pipelinerun_volumes("test-e2e-secret") - vault_vol = next((v for v in volumes if v.get("name") == "vault-secrets"), None) - assert vault_vol is not None, ( - f"Expected a 'vault-secrets' projected volume, got {volumes!r}" - ) - sources = vault_vol.get("projected", {}).get("sources", []) - source_names = [s.get("secret", {}).get("name", "") for s in sources] - assert expected_copy in source_names, ( - f"Projected volume should reference copied secret {expected_copy!r}, " - f"got sources: {source_names!r}" - ) - - copied = core_v1.read_namespaced_secret(expected_copy, NAMESPACE) - owner_refs = copied.metadata.owner_references or [] - assert any( - o.kind == "FournosJob" and o.name == "test-e2e-secret" for o in owner_refs - ), f"Copied secret should have FournosJob ownerRef, got {owner_refs!r}" - assert sorted(copied.data.keys()) == ["password", "username"], ( - f"Copied secret data keys mismatch: {sorted(copied.data.keys())}" - ) - - phase = poll_phase( - k8s, - "test-e2e-secret", - terminal={"Succeeded", "Failed"}, - timeout=60, - ) - assert phase == "Succeeded", job_status_summary(k8s, "test-e2e-secret") - - finally: - _delete_secret_if_exists(core_v1, VAULT_SECRET) - _delete_secret_if_exists(core_v1, expected_copy, namespace=NAMESPACE) - def test_missing_secret_ref_fails(k8s): """A secretRef with no matching labelled Secret fails the job.