From 458b3787493f38394d99abf4d2f6df2d3d884b2b Mon Sep 17 00:00:00 2001 From: Patrick Hermann Date: Sun, 20 Sep 2026 09:27:41 +0000 Subject: [PATCH 1/2] feat(crossplane): let providers carry env and resources, and slow provider-opentofu down Every reconcile of a Workspace is a full tofu init/plan/apply. At the image defaults - poll 10m per resource, sync 1h across all of them - six Workspaces held u26-kind3 at load 8 on 8 cores, tofu and the Proxmox provider binary together around 400% CPU, for Vault configuration that does not change. The cluster paid for it in leader elections: kube-controller-manager, kube-scheduler, crossplane-rbac-manager and openebs each restarted ~3950 times in 39 days, and the machinery play failed intermittently on rollout gates. provider-opentofu now runs with poll 1h / sync 6h and real limits. The values template gained optional env and resources for any provider; a DeploymentRuntimeConfig is emitted when rbac, env or resources are set, so nothing changes for the providers that set none. env rather than args: args in a DeploymentRuntimeConfig replace the image defaults instead of adding to them, and every flag has an env var. Co-Authored-By: Claude Opus 5 (1M context) --- cicd/crossplane-providers.yaml.gotmpl | 21 ++++++++++++++ .../crossplane-providers.values.yaml.gotmpl | 29 +++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/cicd/crossplane-providers.yaml.gotmpl b/cicd/crossplane-providers.yaml.gotmpl index 1aa65646..b51451a8 100644 --- a/cicd/crossplane-providers.yaml.gotmpl +++ b/cicd/crossplane-providers.yaml.gotmpl @@ -44,6 +44,27 @@ environments: package: xpkg.upbound.io/upbound/provider-opentofu:v1.1.6 rbac: clusterRole: cluster-admin + # Every reconcile of a Workspace is a full `tofu init/plan/apply`. + # At the image defaults (poll 10m per resource, sync 1h across all + # of them) six Workspaces held u26-kind3 at load 8 on 8 cores, with + # tofu and the Proxmox provider binary together around 400% CPU, + # for state that does not change. The cluster paid for that in + # leader elections: kube-controller-manager, kube-scheduler, + # crossplane-rbac-manager and openebs each restarted ~3950 times in + # 39 days, because a saturated API server misses lease renewals. + # + # Drift here is Vault configuration nobody edits by hand, so an + # hour between checks costs nothing. + env: + CROSSPLANE_OPENTOFU_PROVIDER_POLL: 1h + CROSSPLANE_OPENTOFU_PROVIDER_SYNC: 6h + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi providerKubeconfig: name: provider-kubeconfig package: ghcr.io/stuttgart-things/provider-kubeconfig-xpkg:v0.13.2 diff --git a/cicd/values/crossplane-providers.values.yaml.gotmpl b/cicd/values/crossplane-providers.values.yaml.gotmpl index 32c03e3d..557c5493 100644 --- a/cicd/values/crossplane-providers.values.yaml.gotmpl +++ b/cicd/values/crossplane-providers.values.yaml.gotmpl @@ -8,22 +8,47 @@ customresources: name: {{ $p.name | required (printf "providers.%s.name is required" $key) }} spec: package: {{ $p.package | required (printf "providers.%s.package is required" $key) }} - {{- if hasKey $p "rbac" }} + {{- if or (hasKey $p "rbac") (hasKey $p "env") (hasKey $p "resources") }} runtimeConfigRef: apiVersion: pkg.crossplane.io/v1beta1 kind: DeploymentRuntimeConfig name: {{ $p.name }} {{- end }} - {{- if hasKey $p "rbac" }} + {{- if or (hasKey $p "rbac") (hasKey $p "env") (hasKey $p "resources") }} {{ $key }}Runtimeconfig: apiVersion: pkg.crossplane.io/v1beta1 kind: DeploymentRuntimeConfig metadata: name: {{ $p.name }} spec: + {{- if hasKey $p "rbac" }} serviceAccountTemplate: metadata: name: {{ $p.name }} + {{- end }} + {{- if or (hasKey $p "env") (hasKey $p "resources") }} + # env, not args: every provider flag also has an environment variable, + # and args in a DeploymentRuntimeConfig REPLACE the image defaults + # instead of adding to them. + deploymentTemplate: + spec: + selector: {} + template: + spec: + containers: + - name: package-runtime + {{- with $p.env }} + env: + {{- range $name, $value := . }} + - name: {{ $name }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- with $p.resources }} + resources: + {{- toYaml . | nindent 20 }} + {{- end }} + {{- end }} {{- end }} {{- end }} From 33baed312f66a51de1e1857d78ff54315441c8d4 Mon Sep 17 00:00:00 2001 From: Patrick Hermann Date: Sun, 20 Sep 2026 15:20:58 +0000 Subject: [PATCH 2/2] feat(crossplane): slow provider-proxmox-bpg down too provider-proxmox-bpg is an upjet provider: every reconcile of a managed resource shells out to terraform with the bpg/proxmox provider binary. At the default poll of 10m and nine Proxmox resources on u26-kind3 that is a terraform run roughly every 70 seconds, and measured on the VM it was the LARGEST single CPU consumer - larger than provider-opentofu, which this branch already slowed down. Same treatment: PROVIDER_POLL 1h, PROVIDER_SYNC 6h and real limits. A VM that Crossplane itself manages does not drift between polls. Co-Authored-By: Claude Opus 5 (1M context) --- cicd/crossplane-providers.yaml.gotmpl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cicd/crossplane-providers.yaml.gotmpl b/cicd/crossplane-providers.yaml.gotmpl index b51451a8..93056210 100644 --- a/cicd/crossplane-providers.yaml.gotmpl +++ b/cicd/crossplane-providers.yaml.gotmpl @@ -83,6 +83,22 @@ environments: providerProxmoxBpg: name: valkiriaaquaticamendi-provider-proxmox-bpg package: xpkg.upbound.io/valkiriaaquaticamendi/provider-proxmox-bpg:v1.18.0 + # An upjet provider: every reconcile of a managed resource shells out + # to terraform with the bpg/proxmox provider binary. At the default + # poll of 10m and nine Proxmox resources on u26-kind3 that is a + # terraform run roughly every 70 seconds, and it was the largest + # single CPU consumer on that VM - larger than provider-opentofu. + # A VM that Crossplane itself manages does not drift between polls. + env: + PROVIDER_POLL: 1h + PROVIDER_SYNC: 6h + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "2" + memory: 2Gi # NOTE: the ClusterProviderConfig resources live in the separate # crossplane-provider-configs.yaml.gotmpl — they reference ProviderConfig # CRDs that only register once these providers are Healthy, so keeping