Skip to content

Commit 2fae758

Browse files
sagtanihampagent
andcommitted
Add the eks install overlay: Amazon S3 snapshots through IRSA
Like the aks overlay, the eks overlay replaces the Google-specific parts of the GKE base for a cluster without certificates.k8s.io/v1beta1, but points atelet at Amazon S3 itself instead of an in-cluster store. Credentials come from IAM Roles for Service Accounts: hack/install-ate.sh annotates the atelet ServiceAccount with eks.amazonaws.com/role-arn (ATE_EKS_ATELET_ROLE_ARN) before the DaemonSet is applied, and publishes the bucket region (ATE_AWS_REGION) in the ate-aws-config ConfigMap that the overlay's atelet reads. The installer defaults --pki-delivery=agent for eks as it does for aks, and only creates rustfs credentials for overlays that ship rustfs. Amp-Thread-ID: https://ampcode.com/threads/T-01a0576a-f26f-71be-a6de-94c1b97b619b Co-authored-by: Amp <amp@ampcode.com>
1 parent ff26921 commit 2fae758

3 files changed

Lines changed: 162 additions & 5 deletions

File tree

‎hack/install-ate.sh‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,10 @@ function usage() {
7777
echo " or the podcert-agent sidecar, for clusters without the"
7878
echo " certificates.k8s.io/v1beta1 API such as AKS and EKS"
7979
echo " --overlay=NAME Install from manifests/ate-install/NAME instead of the GKE base"
80-
echo " (aks: in-cluster S3 snapshot store, no GCP auth; implies --pki-delivery=agent"
81-
echo " unless set). Not combined with kind or --atenet-router=agentgateway"
80+
echo " (aks: in-cluster S3 snapshot store, no GCP auth; eks: Amazon S3 with IRSA,"
81+
echo " needs ATE_AWS_REGION and ATE_EKS_ATELET_ROLE_ARN; both imply"
82+
echo " --pki-delivery=agent unless set). Not combined with kind or"
83+
echo " --atenet-router=agentgateway"
8284
echo " --podcert-workers-per-signer N Concurrent workers per podcertificate-controller signer (default: 1)"
8385
echo " --rollout-timeout DURATION Per-workload readiness wait timeout, kubectl-style Go duration (default: 60s)"
8486
echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)"
@@ -682,6 +684,8 @@ deploy_ate_system() {
682684
fi
683685
fi
684686

687+
annotate_atelet_service_account_for_irsa
688+
685689
local manifests=""
686690
manifests="$(render_ate_system_manifests)"
687691
echo "${manifests}" | run_kubectl apply -f -
@@ -720,10 +724,13 @@ ensure_apiserver_prerequisites() {
720724
create_api_server_env_vars
721725
run_kubectl get configmap -n ate-system ate-api-authentication >/dev/null 2>&1 \
722726
|| create_api_authentication_config
723-
if [[ -n "$(install_overlay)" ]]; then
727+
if [[ -n "$(install_overlay)" && -f "manifests/ate-install/$(install_overlay)/rustfs.yaml" ]]; then
724728
run_kubectl get secret -n ate-system rustfs-credentials >/dev/null 2>&1 \
725729
|| create_rustfs_credentials_secret
726730
fi
731+
if [[ "$(install_overlay)" == "eks" ]]; then
732+
create_aws_config
733+
fi
727734
}
728735

729736
# The in-cluster S3 store of the aks overlay and atelet share these keys
@@ -737,6 +744,40 @@ create_rustfs_credentials_secret() {
737744
--dry-run=client -o yaml | run_kubectl apply -f -
738745
}
739746

747+
# The eks overlay's atelet reads the snapshot bucket's region from this
748+
# ConfigMap (manifests/ate-install/eks/atelet). Reconciled on every install so a
749+
# region change in ATE_AWS_REGION takes effect on the next atelet rollout.
750+
create_aws_config() {
751+
log_step "create_aws_config"
752+
if [[ -z "${ATE_AWS_REGION:-}" ]]; then
753+
echo "Error: --overlay eks needs ATE_AWS_REGION (the region of the S3 snapshot bucket)" >&2
754+
exit 1
755+
fi
756+
run_kubectl create configmap -n ate-system ate-aws-config \
757+
--from-literal=region="${ATE_AWS_REGION}" \
758+
--dry-run=client -o yaml | run_kubectl apply -f -
759+
}
760+
761+
# IRSA for atelet on EKS: the pod identity webhook projects a web identity
762+
# token into pods whose ServiceAccount carries eks.amazonaws.com/role-arn, so
763+
# the annotation must be on the ServiceAccount before the DaemonSet's pods are
764+
# created. The annotation is set imperatively (not through apply) so applying
765+
# the plain manifest afterwards leaves it alone. A changed role needs an atelet
766+
# rollout restart; the installer's apply of the DaemonSet does not restart pods
767+
# on its own when nothing else changed.
768+
annotate_atelet_service_account_for_irsa() {
769+
[[ "$(install_overlay)" == "eks" ]] || return 0
770+
if [[ -z "${ATE_EKS_ATELET_ROLE_ARN:-}" ]]; then
771+
echo "Error: --overlay eks needs ATE_EKS_ATELET_ROLE_ARN (IAM role with access to the S3 snapshot bucket)" >&2
772+
exit 1
773+
fi
774+
log_step "annotate_atelet_service_account_for_irsa"
775+
run_kubectl get serviceaccount -n ate-system atelet >/dev/null 2>&1 \
776+
|| run_kubectl create serviceaccount -n ate-system atelet
777+
run_kubectl annotate serviceaccount -n ate-system atelet --overwrite \
778+
"eks.amazonaws.com/role-arn=${ATE_EKS_ATELET_ROLE_ARN}"
779+
}
780+
740781
# Redeploy only the ate-apiserver
741782
deploy_ate_apiserver() {
742783
log_step "deploy_ate_apiserver"
@@ -764,6 +805,10 @@ deploy_atelet() {
764805

765806
apply_otel_config
766807
apply_otel_endpoint_override
808+
if [[ "$(install_overlay)" == "eks" ]]; then
809+
create_aws_config
810+
fi
811+
annotate_atelet_service_account_for_irsa
767812

768813
local manifest=""
769814
if [[ -n "$(install_overlay)" ]]; then
@@ -1174,8 +1219,8 @@ esac
11741219
podcert_workers_per_signer >/dev/null
11751220
rollout_timeout >/dev/null
11761221
install_overlay >/dev/null
1177-
# The aks overlay exists for clusters without certificates.k8s.io/v1beta1.
1178-
if [[ "${ATE_INSTALL_OVERLAY:-}" == "aks" && -z "${ATE_PKI_DELIVERY:-}" ]]; then
1222+
# The aks and eks overlays exist for clusters without certificates.k8s.io/v1beta1.
1223+
if [[ ("${ATE_INSTALL_OVERLAY:-}" == "aks" || "${ATE_INSTALL_OVERLAY:-}" == "eks") && -z "${ATE_PKI_DELIVERY:-}" ]]; then
11791224
ATE_PKI_DELIVERY=agent
11801225
fi
11811226
if [[ "$(pki_delivery)" == "agent" && "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" == "true" ]]; then
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# atelet for Amazon EKS: no GCP credentials for image pulls, and the S3
16+
# snapshot backend pointed at Amazon S3 itself. The bucket's region comes from
17+
# the ate-aws-config ConfigMap (hack/install-ate.sh creates it from
18+
# ATE_AWS_REGION); the credentials come from IRSA: the installer annotates the
19+
# atelet ServiceAccount with eks.amazonaws.com/role-arn, and EKS's pod identity
20+
# webhook then projects a web identity token and sets AWS_ROLE_ARN and
21+
# AWS_WEB_IDENTITY_TOKEN_FILE on every atelet pod, which the SDK's default
22+
# credential chain in cmd/atelet uses. Nothing here names the account or role.
23+
24+
apiVersion: kustomize.config.k8s.io/v1beta1
25+
kind: Kustomization
26+
27+
resources:
28+
- ../../atelet.yaml
29+
30+
patches:
31+
- patch: |-
32+
apiVersion: apps/v1
33+
kind: DaemonSet
34+
metadata:
35+
name: atelet
36+
namespace: ate-system
37+
spec:
38+
template:
39+
spec:
40+
containers:
41+
- name: atelet
42+
args:
43+
- --gcp-auth-for-image-pulls=false
44+
- --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem
45+
- --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem
46+
- --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem
47+
- --drain-delay=0s
48+
- --drain-timeout=5m
49+
env:
50+
- name: ATE_STORAGE_BACKEND
51+
value: s3
52+
- name: AWS_REGION
53+
valueFrom:
54+
configMapKeyRef:
55+
name: ate-aws-config
56+
key: region
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# eks: the control plane for an Amazon EKS cluster. Like the aks overlay it is
16+
# the GKE base with the Google-specific pieces swapped out, but where aks ships
17+
# an in-cluster S3 store, EKS has the real thing:
18+
#
19+
# - snapshots go to an Amazon S3 bucket; atelet authenticates with IAM Roles
20+
# for Service Accounts (the AWS SDK default credential chain picks up the
21+
# web identity token the EKS pod identity webhook projects once the atelet
22+
# ServiceAccount carries the eks.amazonaws.com/role-arn annotation), so no
23+
# access keys exist anywhere in the cluster. hack/install-ate.sh sets the
24+
# annotation from ATE_EKS_ATELET_ROLE_ARN and the bucket region from
25+
# ATE_AWS_REGION (ConfigMap ate-aws-config), see ./atelet;
26+
# - atelet pulls images without GCP credentials (./atelet);
27+
# - telemetry goes to the small in-cluster OTLP collector the kind overlay
28+
# ships instead of GKE's managed collector;
29+
# - everything else is the base manifest, unchanged. Postgres uses the
30+
# cluster's default StorageClass (create a gp3 one and mark it default on
31+
# clusters that have none, such as EKS Auto Mode).
32+
#
33+
# Certificates: EKS 1.34-1.36 do not serve certificates.k8s.io/v1beta1, so
34+
# install with --pki-delivery=agent (the installer defaults to it for this
35+
# overlay), which layers components/agent-pki on top.
36+
#
37+
# Nodes: the gVisor worker is unprivileged but runsc maps identities in a user
38+
# namespace, so worker nodes need user namespaces enabled
39+
# (user.max_user_namespaces > 0). Amazon Linux 2023 managed node groups
40+
# qualify; EKS Auto Mode's Bottlerocket nodes do not (they set it to 0), so
41+
# run the tainted ate.dev/sandboxClass=gvisor worker pool on a managed node
42+
# group and keep the control plane on whatever nodes the cluster has.
43+
44+
apiVersion: kustomize.config.k8s.io/v1beta1
45+
kind: Kustomization
46+
47+
resources:
48+
- ../ate-api-server.yaml
49+
- ../ate-controller.yaml
50+
- ./atelet
51+
- ../atenet-dns.yaml
52+
- ../atenet-router.yaml
53+
- ../postgres.yaml
54+
- ../pod-certificate-controller.yaml
55+
- ../kind/ate-otel-config.yaml
56+
- ../kind/otel-collector.yaml

0 commit comments

Comments
 (0)