Enforce singleton GPUCluster name and elect ClusterPolicy by creation time

Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>

Author: karthikvetrivel

api/nvidia/v1alpha1/gpucluster_types.go

@@ -186,6 +186,7 @@ type GPUClusterStatus struct {
 //+kubebuilder:resource:scope=Cluster,shortName={"gc"}
 //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,priority=0
 //+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,priority=0
+//+kubebuilder:validation:XValidation:rule="self.metadata.name == 'gpu-cluster'",message="GPUCluster is a singleton, metadata.name must be 'gpu-cluster'"
 
 // GPUCluster is the Schema for the gpuclusters API
 type GPUCluster struct {

bundle/manifests/nvidia.com_gpuclusters.yaml

@@ -1036,6 +1036,9 @@ spec:
             - state
             type: object
         type: object
+        x-kubernetes-validations:
+        - message: GPUCluster is a singleton, metadata.name must be 'gpu-cluster'
+          rule: self.metadata.name == 'gpu-cluster'
     served: true
     storage: true
     subresources:

config/crd/bases/nvidia.com_gpuclusters.yaml

@@ -1036,6 +1036,9 @@ spec:
             - state
             type: object
         type: object
+        x-kubernetes-validations:
+        - message: GPUCluster is a singleton, metadata.name must be 'gpu-cluster'
+          rule: self.metadata.name == 'gpu-cluster'
     served: true
     storage: true
     subresources:

config/samples/nvidia_v1alpha1_gpucluster.yaml

@@ -1,7 +1,7 @@
 apiVersion: nvidia.com/v1alpha1
 kind: GPUCluster
 metadata:
-  name: gpucluster-sample
+  name: gpu-cluster
 spec:
   draDriver:
     repository: registry.k8s.io/dra-driver-nvidia

controllers/active_config.go

@@ -20,35 +20,52 @@ import (
 	"context"
 	"fmt"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
 	"github.com/NVIDIA/gpu-operator/internal/consts"
 )
 
-// TODO: with multiple CRs of a kind, the tie-breaker is list order, which is not
-// guaranteed. Resolve the singleton the reconcilers actually selected instead.
-func resolveActiveConfig(ctx context.Context, c client.Client) (*gpuv1.ClusterPolicy, *nvidiav1alpha1.GPUCluster, error) {
-	var clusterPolicy *gpuv1.ClusterPolicy
+// electSingleton returns the instance that wins the cluster-wide singleton election:
+// oldest creationTimestamp first, lowest name as the tie-breaker. Unlike a
+// first-reconciled-wins claim held in controller memory, the result is stable across
+// operator restarts and derivable by every controller. Returns nil for an empty list.
+func electSingleton[T any, PT interface {
+	*T
+	metav1.Object
+}](items []T) PT {
+	var active PT
+	for i := range items {
+		candidate := PT(&items[i])
+		if active == nil {
+			active = candidate
+			continue
+		}
+		candidateCreated, activeCreated := candidate.GetCreationTimestamp(), active.GetCreationTimestamp()
+		if candidateCreated.Before(&activeCreated) ||
+			(candidateCreated.Equal(&activeCreated) && candidate.GetName() < active.GetName()) {
+			active = candidate
+		}
+	}
+	return active
+}
+
+// resolveActiveConfig returns the active ClusterPolicy and GPUCluster, elected with
+// the same electSingleton rule the owning controllers use.
+func resolveActiveConfig(ctx context.Context, c client.Reader) (*gpuv1.ClusterPolicy, *nvidiav1alpha1.GPUCluster, error) {
 	clusterPolicies := &gpuv1.ClusterPolicyList{}
 	if err := c.List(ctx, clusterPolicies); err != nil {
 		return nil, nil, fmt.Errorf("failed to list ClusterPolicy: %w", err)
 	}
-	if len(clusterPolicies.Items) > 0 {
-		clusterPolicy = &clusterPolicies.Items[0]
-	}
 
-	var gpuCluster *nvidiav1alpha1.GPUCluster
 	gpuClusters := &nvidiav1alpha1.GPUClusterList{}
 	if err := c.List(ctx, gpuClusters); err != nil {
 		return nil, nil, fmt.Errorf("failed to list GPUCluster: %w", err)
 	}
-	if len(gpuClusters.Items) > 0 {
-		gpuCluster = &gpuClusters.Items[0]
-	}
 
-	return clusterPolicy, gpuCluster, nil
+	return electSingleton(clusterPolicies.Items), electSingleton(gpuClusters.Items), nil
 }
 
 // resolveDefaultMode returns the nvidia.com/gpu-operator.resource-allocation.mode value for a GPU node that

controllers/active_config_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -76,6 +77,25 @@ func TestResolveActiveConfig(t *testing.T) {
 		assert.Equal(t, "cluster-config", gc.Name)
 	})
 
+	t.Run("oldest creationTimestamp wins, name breaks ties", func(t *testing.T) {
+		older := &gpuv1.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{
+			Name: "z-older", CreationTimestamp: metav1.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}}
+		newer := &gpuv1.ClusterPolicy{ObjectMeta: metav1.ObjectMeta{
+			Name: "a-newer", CreationTimestamp: metav1.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}}
+		sameTime := metav1.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+		gcB := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "b-cluster", CreationTimestamp: sameTime}}
+		gcA := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "a-cluster", CreationTimestamp: sameTime}}
+		c := fake.NewClientBuilder().WithScheme(scheme).
+			WithObjects(older, newer, gcB, gcA).Build()
+
+		cp, gc, err := resolveActiveConfig(context.Background(), c)
+		require.NoError(t, err)
+		require.NotNil(t, cp)
+		assert.Equal(t, "z-older", cp.Name)
+		require.NotNil(t, gc)
+		assert.Equal(t, "a-cluster", gc.Name)
+	})
+
 	t.Run("neither CR present returns all nil", func(t *testing.T) {
 		c := fake.NewClientBuilder().WithScheme(scheme).Build()
 

controllers/clusterpolicy_controller.go

@@ -117,9 +117,14 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return reconcile.Result{}, err
 	}
 
-	// TODO: Handle deletion of the main ClusterPolicy and cycle to the next one.
-	// We already have a main Clusterpolicy
-	if clusterPolicyCtrl.singleton != nil && clusterPolicyCtrl.singleton.Name != instance.Name {
+	// Singleton election: the ClusterPolicy with the oldest creationTimestamp (name
+	// as tie-breaker) is the active instance, so the choice is stable across
+	// operator restarts and derivable by other controllers (see electSingleton).
+	clusterPolicies := &gpuv1.ClusterPolicyList{}
+	if err := r.List(ctx, clusterPolicies); err != nil {
+		return ctrl.Result{}, fmt.Errorf("error listing ClusterPolicy objects: %w", err)
+	}
+	if active := electSingleton(clusterPolicies.Items); active != nil && active.Name != instance.Name {
 		instance.SetStatus(gpuv1.Ignored, clusterPolicyCtrl.operatorNamespace)
 		// do not change `clusterPolicyCtrl.operatorMetrics.reconciliationStatus` here,
 		// spurious reconciliation

controllers/gpucluster_controller.go

@@ -66,10 +66,6 @@ type GPUClusterReconciler struct {
 	stateManager     state.Manager
 	conditionUpdater conditions.Updater
 	recorder         events.EventRecorder
-
-	// singleton is the GPUCluster that owns reconciliation; the first instance to
-	// reconcile claims it (first-wins), mirroring ClusterPolicy.
-	singleton *nvidiav1alpha1.GPUCluster
 }
 
 //+kubebuilder:rbac:groups=nvidia.com,resources=gpuclusters,verbs=get;list;watch;create;update;patch;delete
@@ -108,18 +104,21 @@ func (r *GPUClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	// stack): every operand DaemonSet of both stacks gates on the per-node
 	// nvidia.com/gpu-operator.resource-allocation.mode label, so each node is served by exactly one stack.
 
-	// Singleton, first-wins (mirroring ClusterPolicy): the first instance to reconcile
-	// claims ownership; any other instance is marked Ignored and skipped. The owner is
-	// held in memory, so the choice resets on operator restart.
-	if r.singleton != nil && r.singleton.Name != instance.Name {
+	// Singleton election (mirroring ClusterPolicy): the GPUCluster with the oldest
+	// creationTimestamp (name as tie-breaker) is the active instance. The CRD's CEL
+	// rule pins metadata.name, so multiple instances can only predate that rule.
+	gpuClusters := &nvidiav1alpha1.GPUClusterList{}
+	if err := r.List(ctx, gpuClusters); err != nil {
+		return ctrl.Result{}, fmt.Errorf("error listing GPUCluster objects: %w", err)
+	}
+	if active := electSingleton(gpuClusters.Items); active != nil && active.Name != instance.Name {
 		logger.V(consts.LogLevelWarning).Info("Multiple GPUCluster instances found, ignoring this one",
-			"name", instance.Name, "owner", r.singleton.Name)
+			"name", instance.Name, "owner", active.Name)
 		if err := r.updateCRStatus(ctx, instance, nvidiav1alpha1.Ignored); err != nil {
 			return ctrl.Result{}, err
 		}
 		return ctrl.Result{}, nil
 	}
-	r.singleton = instance
 
 	// DRA requires all driver management through NVIDIADriver CRs: surface an unmet
 	// prerequisite on this CR's status and hold off deploying operands until it is met.
@@ -185,12 +184,10 @@ func (r *GPUClusterReconciler) validatePrerequisites(ctx context.Context) (strin
 	if err := r.List(ctx, clusterPolicies); err != nil {
 		return "", fmt.Errorf("error listing ClusterPolicy objects: %w", err)
 	}
-	// TODO: check only the active singleton ClusterPolicy once the singleton
-	// selection is resolvable across controllers (see resolveActiveConfig).
-	for _, clusterPolicy := range clusterPolicies.Items {
-		if !clusterPolicy.Spec.Driver.UseNvidiaDriverCRDType() {
-			return fmt.Sprintf("ClusterPolicy %s does not have driver.useNvidiaDriverCRD enabled; migrate driver management to NVIDIADriver CRs before enabling DRA", clusterPolicy.Name), nil
-		}
+	// Only the active singleton ClusterPolicy matters here: an Ignored instance
+	// deploys nothing, so it cannot own driver daemonsets.
+	if active := electSingleton(clusterPolicies.Items); active != nil && !active.Spec.Driver.UseNvidiaDriverCRDType() {
+		return fmt.Sprintf("ClusterPolicy %s does not have driver.useNvidiaDriverCRD enabled; migrate driver management to NVIDIADriver CRs before enabling DRA", active.Name), nil
 	}
 	return "", nil
 }

controllers/gpucluster_controller_test.go

@@ -267,28 +267,31 @@ func TestGPUClusterClusterPolicyDriverPrerequisite(t *testing.T) {
 	require.Equal(t, nvidiav1alpha1.Ready, gccState(t, c, cfg.Name))
 }
 
-// First-reconciled wins (mirroring ClusterPolicy): whichever instance reconciles first
-// claims ownership, regardless of name or creationTimestamp.
+// Oldest creationTimestamp wins the singleton election, regardless of reconcile order.
 func TestGPUClusterSingleton(t *testing.T) {
-	first := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "first"}}
-	second := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "second"}}
-	r, c := newGPUClusterReconciler(t, first, second)
+	older := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{
+		Name: "older", CreationTimestamp: metav1.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}}
+	newer := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{
+		Name: "newer", CreationTimestamp: metav1.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}}
+	r, c := newGPUClusterReconciler(t, older, newer)
 
-	gccReconcile(t, r, first.Name)
-	require.Equal(t, nvidiav1alpha1.Ready, gccState(t, c, first.Name))
+	gccReconcile(t, r, newer.Name)
+	require.Equal(t, nvidiav1alpha1.Ignored, gccState(t, c, newer.Name))
 
-	gccReconcile(t, r, second.Name)
-	require.Equal(t, nvidiav1alpha1.Ignored, gccState(t, c, second.Name))
+	gccReconcile(t, r, older.Name)
+	require.Equal(t, nvidiav1alpha1.Ready, gccState(t, c, older.Name))
 }
 
 // Matching ClusterPolicy, an ignored duplicate carries no status condition.
 func TestGPUClusterDuplicateNoCondition(t *testing.T) {
-	owner := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "owner"}}
-	duplicate := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "duplicate"}}
+	owner := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{
+		Name: "owner", CreationTimestamp: metav1.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}}
+	duplicate := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{
+		Name: "duplicate", CreationTimestamp: metav1.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}}
 	r, c := newGPUClusterReconciler(t, owner, duplicate)
 	r.conditionUpdater = conditions.NewGPUClusterUpdater(c)
 
-	gccReconcile(t, r, owner.Name)     // owner reconciles first, claiming ownership
+	gccReconcile(t, r, owner.Name)     // owner is the older instance and wins the election
 	gccReconcile(t, r, duplicate.Name) // duplicate is ignored
 
 	instance := &nvidiav1alpha1.GPUCluster{}

deployments/gpu-operator/crds/nvidia.com_gpuclusters.yaml

@@ -1036,6 +1036,9 @@ spec:
             - state
             type: object
         type: object
+        x-kubernetes-validations:
+        - message: GPUCluster is a singleton, metadata.name must be 'gpu-cluster'
+          rule: self.metadata.name == 'gpu-cluster'
     served: true
     storage: true
     subresources: