Skip to content

Commit d2fb467

Browse files
cdesiniotiscdesiniotis
committed
Integrate NVIDIA DRA Driver for GPUs as a operand
Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent 47fea81 commit d2fb467

31 files changed

Lines changed: 2308 additions & 9 deletions

‎api/nvidia/v1/clusterpolicy_types.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ type ClusterPolicySpec struct {
5050
Toolkit ToolkitSpec `json:"toolkit"`
5151
// DevicePlugin component spec
5252
DevicePlugin DevicePluginSpec `json:"devicePlugin"`
53+
// DRADriver component spec
54+
DRADriver DRADriverSpec `json:"draDriver"`
5355
// DCGMExporter spec
5456
DCGMExporter DCGMExporterSpec `json:"dcgmExporter"`
5557
// DCGM component spec
@@ -849,6 +851,104 @@ type SandboxDevicePluginSpec struct {
849851
Env []EnvVar `json:"env,omitempty"`
850852
}
851853

854+
// DRADriverSpec defines the properties for the NVIDIA DRA Driver deployment
855+
type DRADriverSpec struct {
856+
// NVIDIA DRA Driver image repository
857+
// +kubebuilder:validation:Optional
858+
Repository string `json:"repository,omitempty"`
859+
860+
// NVIDIA DRA Driver image name
861+
// +kubebuilder:validation:Pattern=[a-zA-Z0-9\-]+
862+
Image string `json:"image,omitempty"`
863+
864+
// NVIDIA DRA Driver image tag
865+
// +kubebuilder:validation:Optional
866+
Version string `json:"version,omitempty"`
867+
868+
// Image pull policy
869+
// +kubebuilder:validation:Optional
870+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
871+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image Pull Policy"
872+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:imagePullPolicy"
873+
ImagePullPolicy string `json:"imagePullPolicy,omitempty"`
874+
875+
// Image pull secrets
876+
// +kubebuilder:validation:Optional
877+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
878+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image pull secrets"
879+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:io.kubernetes:Secret"
880+
ImagePullSecrets []string `json:"imagePullSecrets,omitempty"`
881+
882+
// GPUs defines configuration for GPUs in the NVIDIA DRA Driver
883+
GPUs DRADriverGPUs `json:"gpus,omitempty"`
884+
885+
// ComputeDomains defines configuration for ComputeDomains in the NVIDIA DRA Driver
886+
ComputeDomains DRADriverComputeDomains `json:"computeDomains,omitempty"`
887+
}
888+
889+
// DRADriverGPUs defines configuration for GPUs in the NVIDIA DRA Driver
890+
type DRADriverGPUs struct {
891+
// Enabled indicates if GPUs are enabled in the NVIDIA DRA Driver
892+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
893+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Enable GPUs in the NVIDIA DRA Driver"
894+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
895+
Enabled *bool `json:"enabled,omitempty"`
896+
897+
// KubeletPlugin defines configuration for the NVIDIA DRA Driver kubelet plugin
898+
KubeletPlugin DRADriverKubeletPlugin `json:"kubeletPlugin,omitempty"`
899+
}
900+
901+
// DRADriverComputeDomains defines configuration for ComputeDomains in the NVIDIA DRA Driver
902+
type DRADriverComputeDomains struct {
903+
// Enabled indicates if ComputeDomains are enabled in the NVIDIA DRA Driver
904+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
905+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Enable ComputeDomains in the NVIDIA DRA Driver"
906+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
907+
Enabled *bool `json:"enabled,omitempty"`
908+
909+
// Controller defines configuration for the NVIDIA DRA Driver controller
910+
Controller DRADriverController `json:"controller,omitempty"`
911+
912+
// KubeletPlugin defines configuration for the NVIDIA DRA Driver kubelet plugin
913+
KubeletPlugin DRADriverKubeletPlugin `json:"kubeletPlugin,omitempty"`
914+
}
915+
916+
// DRADriverController defines configuration for the NVIDIA DRA Driver controller
917+
type DRADriverController struct {
918+
// Optional: List of environment variables
919+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
920+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
921+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:text"
922+
Env []EnvVar `json:"env,omitempty"`
923+
924+
// Optional: Define resources requests and limits
925+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
926+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Resource Requirements"
927+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:resourceRequirements"
928+
Resources *ResourceRequirements `json:"resources,omitempty"`
929+
930+
// Optional: Set tolerations
931+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
932+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Tolerations"
933+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:io.kubernetes:Tolerations"
934+
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
935+
}
936+
937+
// DRADriverKubeletPlugin defines configuration for the NVIDIA DRA Driver kubelet plugin
938+
type DRADriverKubeletPlugin struct {
939+
// Optional: List of environment variables
940+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
941+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
942+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:text"
943+
Env []EnvVar `json:"env,omitempty"`
944+
945+
// Optional: Define resources requests and limits
946+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
947+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Resource Requirements"
948+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:resourceRequirements"
949+
Resources *ResourceRequirements `json:"resources,omitempty"`
950+
}
951+
852952
// DCGMExporterSpec defines the properties for NVIDIA DCGM Exporter deployment
853953
type DCGMExporterSpec struct {
854954
// Enabled indicates if deployment of NVIDIA DCGM Exporter through operator is enabled
@@ -1792,6 +1892,9 @@ func ImagePath(spec interface{}) (string, error) {
17921892
case *SandboxDevicePluginSpec:
17931893
config := spec.(*SandboxDevicePluginSpec)
17941894
return imagePath(config.Repository, config.Image, config.Version, "SANDBOX_DEVICE_PLUGIN_IMAGE")
1895+
case *DRADriverSpec:
1896+
config := spec.(*DRADriverSpec)
1897+
return imagePath(config.Repository, config.Image, config.Version, "DRA_DRIVER_IMAGE")
17951898
case *DCGMExporterSpec:
17961899
config := spec.(*DCGMExporterSpec)
17971900
return imagePath(config.Repository, config.Image, config.Version, "DCGM_EXPORTER_IMAGE")
@@ -1896,6 +1999,21 @@ func (p *DevicePluginSpec) IsEnabled() bool {
18961999
return *p.Enabled
18972000
}
18982001

2002+
// IsEnabled returns true if the DRA Driver is enabled through gpu-operator
2003+
func (d *DRADriverSpec) IsEnabled() bool {
2004+
return d.IsGPUsEnabled() || d.IsComputeDomainsEnabled()
2005+
}
2006+
2007+
// IsGPUsEnabled returns true if the GPUs resource is enabled in the DRA Driver
2008+
func (d *DRADriverSpec) IsGPUsEnabled() bool {
2009+
return d.GPUs.Enabled != nil && *d.GPUs.Enabled
2010+
}
2011+
2012+
// IsComputeDomainsEnabled returns true if the ComputeDomains resource is enabled in the DRA Driver
2013+
func (d *DRADriverSpec) IsComputeDomainsEnabled() bool {
2014+
return d.ComputeDomains.Enabled != nil && *d.ComputeDomains.Enabled
2015+
}
2016+
18992017
// IsEnabled returns true if dcgm-exporter is enabled(default) through gpu-operator
19002018
func (e *DCGMExporterSpec) IsEnabled() bool {
19012019
if e.Enabled == nil {

‎api/nvidia/v1/zz_generated.deepcopy.go‎

Lines changed: 123 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/state-dra-driver/0100_service_account.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: nvidia-dra-driver
5+
namespace: "FILLED BY THE OPERATOR"

‎assets/state-dra-driver/0200_clusterrole.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: nvidia-dra-driver
5+
namespace: "FILLED BY THE OPERATOR"
6+
rules:
7+
- apiGroups: ["resource.nvidia.com"]
8+
resources: ["computedomains"]
9+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
10+
- apiGroups: ["resource.nvidia.com"]
11+
resources: ["computedomains/status"]
12+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
13+
- apiGroups: ["resource.k8s.io"]
14+
resources: ["resourceclaims"]
15+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
16+
- apiGroups: ["resource.k8s.io"]
17+
resources: ["resourceclaimtemplates"]
18+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
19+
- apiGroups: ["resource.k8s.io"]
20+
resources: ["deviceclasses"]
21+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
22+
- apiGroups: ["resource.k8s.io"]
23+
resources: ["resourceslices"]
24+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
25+
- apiGroups: ["resource.k8s.io"]
26+
resources: ["resourceclaims/status"]
27+
verbs: ["update"]
28+
- apiGroups: [""]
29+
resources: ["nodes"]
30+
verbs: ["get", "list", "watch", "update", "patch"]
31+
- apiGroups: [""]
32+
resources: ["pods"]
33+
verbs: ["get", "list", "watch"]

‎assets/state-dra-driver/0210_role.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: Role
3+
metadata:
4+
name: nvidia-dra-driver
5+
namespace: "FILLED BY THE OPERATOR"
6+
rules:
7+
- apiGroups: ["apps"]
8+
resources: ["daemonsets", "deployments"]
9+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

‎assets/state-dra-driver/0300_clusterrolebinding.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: nvidia-dra-driver
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: nvidia-dra-driver
9+
subjects:
10+
- kind: ServiceAccount
11+
name: nvidia-dra-driver
12+
namespace: "FILLED BY THE OPERATOR"

‎assets/state-dra-driver/0310_rolebinding.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: RoleBinding
3+
metadata:
4+
name: nvidia-dra-driver
5+
namespace: "FILLED BY THE OPERATOR"
6+
roleRef:
7+
apiGroup: rbac.authorization.k8s.io
8+
kind: Role
9+
name: nvidia-dra-driver
10+
subjects:
11+
- kind: ServiceAccount
12+
name: nvidia-dra-driver
13+
namespace: "FILLED BY THE OPERATOR"

‎assets/state-dra-driver/0400_deviceclass-compute-domain-daemon.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: resource.k8s.io/v1beta1
2+
kind: DeviceClass
3+
metadata:
4+
name: compute-domain-daemon.nvidia.com
5+
spec:
6+
selectors:
7+
- cel:
8+
expression: "device.driver == 'compute-domain.nvidia.com' && device.attributes['compute-domain.nvidia.com'].type == 'daemon'"

‎assets/state-dra-driver/0410_deviceclass-compute-domain-default-channel.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: resource.k8s.io/v1beta1
2+
kind: DeviceClass
3+
metadata:
4+
name: compute-domain-default-channel.nvidia.com
5+
spec:
6+
selectors:
7+
- cel:
8+
expression: "device.driver == 'compute-domain.nvidia.com' && device.attributes['compute-domain.nvidia.com'].type == 'channel' && device.attributes['compute-domain.nvidia.com'].id == 0"

‎assets/state-dra-driver/0420_deviceclass-gpu.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: resource.k8s.io/v1beta1
2+
kind: DeviceClass
3+
metadata:
4+
name: gpu.nvidia.com
5+
spec:
6+
selectors:
7+
- cel:
8+
expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].type == 'gpu'"

0 commit comments

Comments
 (0)