Skip to content

Commit bfff07b

Browse files
cdesiniotiscdesiniotis
committed
Add DRA driver for IMEX
Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent fe9595a commit bfff07b

17 files changed

Lines changed: 738 additions & 2 deletions

‎api/nvidia/v1/clusterpolicy_types.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ type ClusterPolicySpec struct {
5353
Toolkit ToolkitSpec `json:"toolkit"`
5454
// DevicePlugin component spec
5555
DevicePlugin DevicePluginSpec `json:"devicePlugin"`
56+
// DRADriver component spec
57+
DRADriver DRADriverSpec `json:"draDriver"`
5658
// DCGMExporter spec
5759
DCGMExporter DCGMExporterSpec `json:"dcgmExporter"`
5860
// DCGM component spec
@@ -841,6 +843,60 @@ type SandboxDevicePluginSpec struct {
841843
Env []EnvVar `json:"env,omitempty"`
842844
}
843845

846+
// DRADriverSpec defines the properties for the NVIDIA DRA Driver deployment
847+
// TODO: add 'controller' and 'kubeletPlugin' structs to allow for per-component configuration
848+
type DRADriverSpec struct {
849+
// Enabled indicates if the deployment of NVIDIA DRA Driver through the operator is enabled
850+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
851+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Enable NVIDIA DRA Driver deployment through GPU Operator"
852+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
853+
Enabled *bool `json:"enabled,omitempty"`
854+
855+
// NVIDIA DRA Driver image repository
856+
// +kubebuilder:validation:Optional
857+
Repository string `json:"repository,omitempty"`
858+
859+
// NVIDIA DRA Driver image name
860+
// +kubebuilder:validation:Pattern=[a-zA-Z0-9\-]+
861+
Image string `json:"image,omitempty"`
862+
863+
// NVIDIA DRA Driver image tag
864+
// +kubebuilder:validation:Optional
865+
Version string `json:"version,omitempty"`
866+
867+
// Image pull policy
868+
// +kubebuilder:validation:Optional
869+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
870+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image Pull Policy"
871+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:imagePullPolicy"
872+
ImagePullPolicy string `json:"imagePullPolicy,omitempty"`
873+
874+
// Image pull secrets
875+
// +kubebuilder:validation:Optional
876+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
877+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Image pull secrets"
878+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:io.kubernetes:Secret"
879+
ImagePullSecrets []string `json:"imagePullSecrets,omitempty"`
880+
881+
// Optional: Define resources requests and limits for each pod
882+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
883+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Resource Requirements"
884+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:resourceRequirements"
885+
Resources *ResourceRequirements `json:"resources,omitempty"`
886+
887+
// Optional: List of arguments
888+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
889+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Arguments"
890+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:text"
891+
Args []string `json:"args,omitempty"`
892+
893+
// Optional: List of environment variables
894+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
895+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Environment Variables"
896+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:text"
897+
Env []EnvVar `json:"env,omitempty"`
898+
}
899+
844900
// DCGMExporterSpec defines the properties for NVIDIA DCGM Exporter deployment
845901
type DCGMExporterSpec struct {
846902
// Enabled indicates if deployment of NVIDIA DCGM Exporter through operator is enabled
@@ -1764,6 +1820,9 @@ func ImagePath(spec interface{}) (string, error) {
17641820
case *SandboxDevicePluginSpec:
17651821
config := spec.(*SandboxDevicePluginSpec)
17661822
return imagePath(config.Repository, config.Image, config.Version, "SANDBOX_DEVICE_PLUGIN_IMAGE")
1823+
case *DRADriverSpec:
1824+
config := spec.(*DRADriverSpec)
1825+
return imagePath(config.Repository, config.Image, config.Version, "DRA_DRIVER_IMAGE")
17671826
case *DCGMExporterSpec:
17681827
config := spec.(*DCGMExporterSpec)
17691828
return imagePath(config.Repository, config.Image, config.Version, "DCGM_EXPORTER_IMAGE")
@@ -1872,6 +1931,15 @@ func (p *DevicePluginSpec) IsEnabled() bool {
18721931
return *p.Enabled
18731932
}
18741933

1934+
// IsEnabled returns true if draDriver is enabled through gpu-operator
1935+
func (d *DRADriverSpec) IsEnabled() bool {
1936+
if d.Enabled == nil {
1937+
// default is true if not specified by user
1938+
return true
1939+
}
1940+
return *d.Enabled
1941+
}
1942+
18751943
// IsEnabled returns true if dcgm-exporter is enabled(default) through gpu-operator
18761944
func (e *DCGMExporterSpec) IsEnabled() bool {
18771945
if e.Enabled == nil {

‎api/nvidia/v1/zz_generated.deepcopy.go‎

Lines changed: 41 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎assets/state-dra-driver/0100_service_account.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: nvidia-dra-driver
5+
namespace: "FILLED BY THE OPERATOR"

‎assets/state-dra-driver/0200_clusterrole.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: nvidia-dra-driver
5+
rules:
6+
# TODO: restrict RBAC for DRA driver
7+
- apiGroups:
8+
- ""
9+
- apps
10+
- resource.k8s.io
11+
- gpu.nvidia.com
12+
resources:
13+
- '*'
14+
verbs:
15+
- '*'

‎assets/state-dra-driver/0300_clusterrolebinding.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: nvidia-dra-driver
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: nvidia-dra-driver
9+
subjects:
10+
- kind: ServiceAccount
11+
name: nvidia-dra-driver
12+
namespace: "FILLED BY THE OPERATOR"

‎assets/state-dra-driver/0400_deviceclass-imex.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: resource.k8s.io/v1alpha3
2+
kind: DeviceClass
3+
metadata:
4+
name: imex.nvidia.com
5+
spec:
6+
selectors:
7+
- cel:
8+
expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].type == 'imex-channel'"

‎assets/state-dra-driver/0500_deployment.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
app: nvidia-dra-driver-controller
6+
name: nvidia-dra-driver-controller
7+
namespace: "FILLED BY THE OPERATOR"
8+
spec:
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: nvidia-dra-driver-controller
13+
template:
14+
metadata:
15+
labels:
16+
app: nvidia-dra-driver-controller
17+
spec:
18+
priorityClassName: system-node-critical
19+
serviceAccountName: nvidia-dra-driver
20+
tolerations:
21+
- effect: NoSchedule
22+
key: node-role.kubernetes.io/master
23+
operator: Exists
24+
- effect: NoSchedule
25+
key: node-role.kubernetes.io/control-plane
26+
operator: Exists
27+
containers:
28+
- name: controller
29+
image: "FILLED BY THE OPERATOR"
30+
imagePullPolicy: IfNotPresent
31+
command: ["nvidia-dra-controller", "-v", "6"]
32+
env:
33+
- name: DEVICE_CLASSES
34+
value: imex
35+
- name: POD_NAME
36+
valueFrom:
37+
fieldRef:
38+
apiVersion: v1
39+
fieldPath: metadata.name
40+
- name: NAMESPACE
41+
valueFrom:
42+
fieldRef:
43+
apiVersion: v1
44+
fieldPath: metadata.namespace

‎assets/state-dra-driver/0600_configmap.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: nvidia-dra-driver-kubelet-plugin-entrypoint
5+
namespace: "FILLED BY THE OPERATOR"
6+
labels:
7+
app: nvidia-dra-driver-kubelet-plugin
8+
data:
9+
entrypoint.sh: |-
10+
#!/bin/bash
11+
12+
until [[ -f /run/nvidia/validations/driver-ready ]]
13+
do
14+
echo "waiting for the driver validations to be ready..."
15+
sleep 5
16+
done
17+
18+
set -o allexport
19+
cat /run/nvidia/validations/driver-ready
20+
. /run/nvidia/validations/driver-ready
21+
# TODO: add an alias for DRIVER_ROOT_CTR_PATH in the k8s-dra-driver and remove the below export
22+
export CONTAINER_DRIVER_ROOT=$DRIVER_ROOT_CTR_PATH
23+
24+
# Conditionally mask the params file to prevent this container from
25+
# recreating any missing GPU device nodes. This is necessary, for
26+
# example, when running under nvkind to limit the set GPUs governed
27+
# by the plugin even though it has cgroup access to all of them.
28+
if [ "${MASK_NVIDIA_DRIVER_PARAMS}" = "true" ]; then
29+
cp /proc/driver/nvidia/params root/gpu-params
30+
sed -i 's/^ModifyDeviceFiles: 1$/ModifyDeviceFiles: 0/' root/gpu-params
31+
mount --bind root/gpu-params /proc/driver/nvidia/params
32+
fi
33+
echo "Starting nvidia-dra-plugin"
34+
exec nvidia-dra-plugin

0 commit comments

Comments
 (0)