Skip to content

Commit 5d83cd2

Browse files
committed
[gateway] support direct HTTPRoute backendRefs to VPCService
Routes can now reference a VPCService directly via a backendRef with group vpc.apoxy.dev and kind VPCService, without a wrapper Backend object. The translator resolves the reference to the service's overlay FQDN (<hostname>.<network>.vpc.apoxy.net) and marks the destination InputDerived so the datapath patcher applies as before. Add spec.appProtocol to VPCService with the GEP-1911 vocabulary (kubernetes.io/h2c, grpc) to select the upstream protocol; empty means HTTP/1.1. The dedicated-mode gateway controller gathers referenced VPCServices with per-route-kind field indexes and retranslates on spec changes.
1 parent bb1d6e6 commit 5d83cd2

10 files changed

Lines changed: 381 additions & 3 deletions

File tree

api/generated/zz_generated.openapi.go

Lines changed: 8 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

api/vpc/v1alpha1/vpcservice_types.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,24 @@ type VPCServiceSpec struct {
3434
// connect; the relay stamps them onto Tunnel metadata labels).
3535
// +required
3636
Selector *metav1.LabelSelector `json:"selector"`
37+
38+
// The application protocol the members speak, using the Gateway API
39+
// (GEP-1911) vocabulary: "kubernetes.io/h2c" for cleartext HTTP/2 and
40+
// "grpc" for gRPC (which implies h2c). Empty means HTTP/1.1. Routes that
41+
// reference this service use it to pick the upstream protocol.
42+
// +optional
43+
AppProtocol string `json:"appProtocol,omitempty"`
3744
}
3845

46+
// Application protocol values accepted in spec.appProtocol.
47+
const (
48+
// AppProtocolH2C selects cleartext HTTP/2 (GEP-1911 standard value).
49+
AppProtocolH2C = "kubernetes.io/h2c"
50+
51+
// AppProtocolGRPC selects gRPC, which is carried over cleartext HTTP/2.
52+
AppProtocolGRPC = "grpc"
53+
)
54+
3955
// MembershipSelector converts spec.selector into the label selector used to
4056
// pick member Tunnels, and is the single definition of which selectors are
4157
// usable as a membership rule.

api/vpc/v1alpha1/vpcservice_validate.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,15 @@ func (s *VPCService) validate() field.ErrorList {
8787
}
8888
}
8989

90+
// The protocol vocabulary is closed (GEP-1911): a value the route
91+
// translator does not know would silently fall back to HTTP/1.1, which
92+
// for a gRPC backend is a runtime failure with no admission-time signal.
93+
switch s.Spec.AppProtocol {
94+
case "", AppProtocolH2C, AppProtocolGRPC:
95+
default:
96+
errs = append(errs, field.NotSupported(field.NewPath("spec", "appProtocol"),
97+
s.Spec.AppProtocol, []string{AppProtocolH2C, AppProtocolGRPC}))
98+
}
99+
90100
return errs
91101
}

pkg/apiserver/gateway/gateway.go

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
extensionsv1alpha2 "github.com/apoxy-dev/apoxy/api/extensions/v1alpha2"
3333
gatewayv1 "github.com/apoxy-dev/apoxy/api/gateway/v1"
3434
gatewayv1alpha2 "github.com/apoxy-dev/apoxy/api/gateway/v1alpha2"
35+
vpcv1alpha1 "github.com/apoxy-dev/apoxy/api/vpc/v1alpha1"
3536
)
3637

3738
func Install(scheme *runtime.Scheme) {
@@ -54,6 +55,11 @@ const (
5455
serviceTLSRouteIndex = "serviceTLSRouteIndex"
5556
gatewayInfraRefIndex = "gatewayInfraRefIndex"
5657
edgeFunctionLiveIndex = "edgeFunctionLiveIndex"
58+
59+
vpcServiceHTTPRouteIndex = "vpcServiceHTTPRouteIndex"
60+
vpcServiceTCPRouteIndex = "vpcServiceTCPRouteIndex"
61+
vpcServiceUDPRouteIndex = "vpcServiceUDPRouteIndex"
62+
vpcServiceTLSRouteIndex = "vpcServiceTLSRouteIndex"
5763
)
5864

5965
var (
@@ -173,6 +179,9 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, request reconcile.Req
173179
if err := r.reconcileBackends(clog.IntoContext(ctx, log), res); err != nil {
174180
log.Error(err, "Failed to reconcile BackendRefs for GatewayClass", "name", gwc.Name)
175181
}
182+
if err := r.reconcileVPCServices(clog.IntoContext(ctx, log), res); err != nil {
183+
log.Error(err, "Failed to reconcile VPCServices for GatewayClass", "name", gwc.Name)
184+
}
176185
if r.watchK8s {
177186
if err := r.reconcileServices(clog.IntoContext(ctx, log), res); err != nil {
178187
log.Error(err, "Failed to reconcile Services for GatewayClass", "name", gwc.Name)
@@ -616,6 +625,66 @@ func (r *GatewayReconciler) reconcileBackends(
616625
return nil
617626
}
618627

628+
// reconcileVPCServices collects the VPCServices referenced by at least one
629+
// route into the translation snapshot. The translator only reads the spec
630+
// (hostname, networkRef, appProtocol) to mint the vpc-zone FQDN; membership
631+
// changes flow through the endpoint plane and do not pass through here.
632+
func (r *GatewayReconciler) reconcileVPCServices(
633+
ctx context.Context,
634+
res *gatewayapi.Resources,
635+
) error {
636+
log := clog.FromContext(ctx)
637+
638+
var vsl vpcv1alpha1.VPCServiceList
639+
if err := r.List(ctx, &vsl); err != nil {
640+
return fmt.Errorf("failed to list VPCServices: %w", err)
641+
}
642+
643+
for _, s := range vsl.Items {
644+
if !s.DeletionTimestamp.IsZero() {
645+
log.Info("VPCService is being deleted", "name", s.Name)
646+
continue
647+
}
648+
649+
var hasRouteRef bool
650+
651+
var hrsl gatewayv1.HTTPRouteList
652+
if err := r.List(ctx, &hrsl, client.MatchingFields{vpcServiceHTTPRouteIndex: string(s.Name)}); err != nil {
653+
return fmt.Errorf("failed to list HTTPRoutes for VPCService %s: %w", s.Name, err)
654+
}
655+
hasRouteRef = hasRouteRef || len(hrsl.Items) > 0
656+
657+
var trsl gatewayv1alpha2.TCPRouteList
658+
if err := r.List(ctx, &trsl, client.MatchingFields{vpcServiceTCPRouteIndex: string(s.Name)}); err != nil {
659+
return fmt.Errorf("failed to list TCPRoutes for VPCService %s: %w", s.Name, err)
660+
}
661+
hasRouteRef = hasRouteRef || len(trsl.Items) > 0
662+
663+
var ursl gatewayv1alpha2.UDPRouteList
664+
if err := r.List(ctx, &ursl, client.MatchingFields{vpcServiceUDPRouteIndex: string(s.Name)}); err != nil {
665+
return fmt.Errorf("failed to list UDPRoutes for VPCService %s: %w", s.Name, err)
666+
}
667+
hasRouteRef = hasRouteRef || len(ursl.Items) > 0
668+
669+
var tlsrsl gatewayv1alpha2.TLSRouteList
670+
if err := r.List(ctx, &tlsrsl, client.MatchingFields{vpcServiceTLSRouteIndex: string(s.Name)}); err != nil {
671+
return fmt.Errorf("failed to list TLSRoutes for VPCService %s: %w", s.Name, err)
672+
}
673+
hasRouteRef = hasRouteRef || len(tlsrsl.Items) > 0
674+
675+
if !hasRouteRef {
676+
log.V(1).Info("No matching Route objects found for VPCService", "name", s.Name)
677+
continue
678+
}
679+
680+
log.V(1).Info("Reconciling VPCService", "name", s.Name)
681+
682+
res.VPCServices = append(res.VPCServices, &s)
683+
}
684+
685+
return nil
686+
}
687+
619688
func (r *GatewayReconciler) reconcileServices(
620689
ctx context.Context,
621690
res *gatewayapi.Resources,
@@ -845,6 +914,64 @@ func (r *GatewayReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
845914
return fmt.Errorf("failed to setup field indexer: %w", err)
846915
}
847916
}
917+
// Indexes each route type by the name of the referenced VPCService
918+
// object, mirroring the Backend indexes above.
919+
if err := mgr.GetFieldIndexer().IndexField(ctx, &gatewayv1.HTTPRoute{}, vpcServiceHTTPRouteIndex, func(obj client.Object) []string {
920+
route := obj.(*gatewayv1.HTTPRoute)
921+
var services []string
922+
for _, rule := range route.Spec.Rules {
923+
for _, backend := range rule.BackendRefs {
924+
if backend.Kind != nil && *backend.Kind == "VPCService" {
925+
services = append(services, string(backend.Name))
926+
}
927+
}
928+
}
929+
return services
930+
}); err != nil {
931+
return fmt.Errorf("failed to setup field indexer: %w", err)
932+
}
933+
if err := mgr.GetFieldIndexer().IndexField(ctx, &gatewayv1alpha2.TCPRoute{}, vpcServiceTCPRouteIndex, func(obj client.Object) []string {
934+
route := obj.(*gatewayv1alpha2.TCPRoute)
935+
var services []string
936+
for _, rule := range route.Spec.Rules {
937+
for _, backend := range rule.BackendRefs {
938+
if backend.Kind != nil && *backend.Kind == "VPCService" {
939+
services = append(services, string(backend.Name))
940+
}
941+
}
942+
}
943+
return services
944+
}); err != nil {
945+
return fmt.Errorf("failed to setup field indexer: %w", err)
946+
}
947+
if err := mgr.GetFieldIndexer().IndexField(ctx, &gatewayv1alpha2.UDPRoute{}, vpcServiceUDPRouteIndex, func(obj client.Object) []string {
948+
route := obj.(*gatewayv1alpha2.UDPRoute)
949+
var services []string
950+
for _, rule := range route.Spec.Rules {
951+
for _, backend := range rule.BackendRefs {
952+
if backend.Kind != nil && *backend.Kind == "VPCService" {
953+
services = append(services, string(backend.Name))
954+
}
955+
}
956+
}
957+
return services
958+
}); err != nil {
959+
return fmt.Errorf("failed to setup field indexer: %w", err)
960+
}
961+
if err := mgr.GetFieldIndexer().IndexField(ctx, &gatewayv1alpha2.TLSRoute{}, vpcServiceTLSRouteIndex, func(obj client.Object) []string {
962+
route := obj.(*gatewayv1alpha2.TLSRoute)
963+
var services []string
964+
for _, rule := range route.Spec.Rules {
965+
for _, backend := range rule.BackendRefs {
966+
if backend.Kind != nil && *backend.Kind == "VPCService" {
967+
services = append(services, string(backend.Name))
968+
}
969+
}
970+
}
971+
return services
972+
}); err != nil {
973+
return fmt.Errorf("failed to setup field indexer: %w", err)
974+
}
848975
// Index EdgeFunction objects that are ready.
849976
if err := mgr.GetFieldIndexer().IndexField(ctx, &extensionsv1alpha2.EdgeFunction{}, edgeFunctionLiveIndex, func(obj client.Object) []string {
850977
if obj.(*extensionsv1alpha2.EdgeFunction).Status.LiveRevision != "" {
@@ -905,6 +1032,11 @@ func (r *GatewayReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
9051032
handler.EnqueueRequestsFromMapFunc(r.enqueueClass),
9061033
builder.WithPredicates(generationOrDeletion),
9071034
).
1035+
Watches(
1036+
&vpcv1alpha1.VPCService{},
1037+
handler.EnqueueRequestsFromMapFunc(r.enqueueClass),
1038+
builder.WithPredicates(generationOrDeletion),
1039+
).
9081040
Watches(
9091041
&extensionsv1alpha2.EdgeFunction{},
9101042
handler.EnqueueRequestsFromMapFunc(r.enqueueClass),

pkg/gateway/gatewayapi/resource.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
corev1alpha2 "github.com/apoxy-dev/apoxy/api/core/v1alpha2"
2424
extensionsv1alpha2 "github.com/apoxy-dev/apoxy/api/extensions/v1alpha2"
25+
vpcv1alpha1 "github.com/apoxy-dev/apoxy/api/vpc/v1alpha1"
2526
)
2627

2728
const (
@@ -63,6 +64,7 @@ type Resources struct {
6364
EdgeFunctionBackends []*extensionsv1alpha2.EdgeFunction `json:"edgeFunctionBackends,omitempty" yaml:"edgeFunctionBackends,omitempty"`
6465
EdgeFunctionRevisions []*extensionsv1alpha2.EdgeFunctionRevision `json:"edgeFunctionFilters,omitempty" yaml:"edgeFunctionFilters,omitempty"`
6566
Backends []*corev1alpha2.Backend `json:"backends,omitempty" yaml:"backends,omitempty"`
67+
VPCServices []*vpcv1alpha1.VPCService `json:"vpcServices,omitempty" yaml:"vpcServices,omitempty"`
6668
Proxies []*corev1alpha2.Proxy `json:"proxies,omitempty" yaml:"proxies,omitempty"`
6769
DirectResponses []*extensionsv1alpha2.DirectResponse `json:"directResponses,omitempty" yaml:"directResponses,omitempty"`
6870
}
@@ -171,6 +173,15 @@ func (r *Resources) GetBackend(name string) *corev1alpha2.Backend {
171173
return nil
172174
}
173175

176+
func (r *Resources) GetVPCService(name string) *vpcv1alpha1.VPCService {
177+
for _, svc := range r.VPCServices {
178+
if svc.Name == name {
179+
return svc
180+
}
181+
}
182+
return nil
183+
}
184+
174185
func (r *Resources) GetProxy(name string) (*corev1alpha2.Proxy, bool) {
175186
for _, proxy := range r.Proxies {
176187
if proxy.Name == name {

pkg/gateway/gatewayapi/route.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
v1alpha2 "github.com/apoxy-dev/apoxy/api/core/v1alpha2"
3131
extensionsv1alpha2 "github.com/apoxy-dev/apoxy/api/extensions/v1alpha2"
3232
agwapiv1a1 "github.com/apoxy-dev/apoxy/api/gateway/v1"
33+
vpcv1alpha1 "github.com/apoxy-dev/apoxy/api/vpc/v1alpha1"
3334
)
3435

3536
const (
@@ -1355,6 +1356,19 @@ func (t *Translator) processDestination(
13551356
return nil
13561357
}
13571358
ds.Filters = t.processDestinationFilters(routeType, backendRefContext, parentRef, route, resources)
1359+
case KindVPCService:
1360+
var err error
1361+
ds, err = t.processVPCServiceDestinationSetting(backendRef.BackendObjectReference, protocol, resources)
1362+
if err != nil {
1363+
log.Errorf("failed to process VPCService ref %s: %v", backendRef.Name, err)
1364+
parentRef.SetCondition(route,
1365+
gwapiv1.RouteConditionResolvedRefs,
1366+
metav1.ConditionFalse,
1367+
gwapiv1a2.RouteReasonResolvedRefs,
1368+
err.Error())
1369+
return nil
1370+
}
1371+
ds.Filters = t.processDestinationFilters(routeType, backendRefContext, parentRef, route, resources)
13581372
case KindEdgeFunction:
13591373
log.DefaultLogger.Debug("Processing edge function backend ref", "name", backendRef.Name)
13601374

@@ -1702,6 +1716,49 @@ func (t *Translator) processBackendDestinationSetting(backendRef gwapiv1.Backend
17021716
return &ds, nil
17031717
}
17041718

1719+
// processVPCServiceDestinationSetting resolves a backendRef of kind
1720+
// VPCService into the service's vpc-zone FQDN
1721+
// (<hostname>.<network>.vpc.apoxy.net). The FQDN is the whole contract: the
1722+
// translator emits a STRICT_DNS cluster for it and the cloud xDS patch layer
1723+
// rewrites vpc-zone clusters to overlay EDS. The service's member endpoints
1724+
// are therefore not read here. The upstream protocol comes from
1725+
// spec.appProtocol (GEP-1911 vocabulary); the port must come from the
1726+
// backendRef because a VPCService carries no port of its own.
1727+
func (t *Translator) processVPCServiceDestinationSetting(
1728+
backendRef gwapiv1.BackendObjectReference,
1729+
protocol ir.AppProtocol,
1730+
resources *Resources,
1731+
) (*ir.DestinationSetting, error) {
1732+
if backendRef.Port == nil {
1733+
return nil, errors.New("port is required for VPCService reference")
1734+
}
1735+
port := int(*backendRef.Port)
1736+
if port < 1 || port > 65535 {
1737+
return nil, fmt.Errorf("invalid port %d", port)
1738+
}
1739+
svc := resources.GetVPCService(string(backendRef.Name))
1740+
if svc == nil {
1741+
return nil, fmt.Errorf("VPCService %q not found", backendRef.Name)
1742+
}
1743+
1744+
switch svc.Spec.AppProtocol {
1745+
case vpcv1alpha1.AppProtocolH2C:
1746+
protocol = ir.HTTP2
1747+
case vpcv1alpha1.AppProtocolGRPC:
1748+
protocol = ir.GRPC
1749+
}
1750+
1751+
return &ir.DestinationSetting{
1752+
InputDerived: true,
1753+
Protocol: protocol,
1754+
AddressType: ptr.To(ir.FQDN),
1755+
Endpoints: []*ir.DestinationEndpoint{{
1756+
Host: apoxynet.VPCServiceFQDN(svc.DNSHostname(), svc.Spec.NetworkRef.Name),
1757+
Port: uint32(port),
1758+
}},
1759+
}, nil
1760+
}
1761+
17051762
func (t *Translator) processEdgeFunctionDestinationSetting(
17061763
backendRef gwapiv1.BackendObjectReference,
17071764
protocol ir.AppProtocol,

pkg/gateway/gatewayapi/translator.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,15 @@ const (
3030
KindService = "Service"
3131
KindServiceImport = "ServiceImport"
3232
KindBackend = "Backend"
33+
KindVPCService = "VPCService"
3334
KindEdgeFunction = "EdgeFunction"
3435
KindSecret = "Secret"
3536
KindSecurityPolicy = "SecurityPolicy"
3637

3738
GroupApoxyCore = "core.apoxy.dev"
3839
GroupApoxyExtensions = "extensions.apoxy.dev"
3940
GroupApoxyCompute = "compute.apoxy.dev"
41+
GroupApoxyVPC = "vpc.apoxy.dev"
4042
GroupMultiClusterService = "multicluster.x-k8s.io"
4143

4244
// OwningGatewayNamespaceLabel is the owner reference label used for managed infra.

pkg/gateway/gatewayapi/validate.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (t *Translator) validateBackendRef(backendRefContext BackendRefContext, par
6969
}
7070

7171
func (t *Translator) validateBackendRefGroup(backendRef *gwapiv1a2.BackendRef, parentRef *RouteParentContext, route RouteContext) bool {
72-
accepted := []string{GroupApoxyCore, GroupApoxyExtensions, GroupApoxyCompute, GroupMultiClusterService}
72+
accepted := []string{GroupApoxyCore, GroupApoxyExtensions, GroupApoxyCompute, GroupApoxyVPC, GroupMultiClusterService}
7373
if backendRef.Group != nil &&
7474
*backendRef.Group != "" &&
7575
!sets.NewString(accepted...).Has(string(*backendRef.Group)) {
@@ -86,7 +86,7 @@ func (t *Translator) validateBackendRefGroup(backendRef *gwapiv1a2.BackendRef, p
8686
}
8787

8888
func (t *Translator) validateBackendRefKind(backendRef *gwapiv1a2.BackendRef, parentRef *RouteParentContext, route RouteContext) bool {
89-
accepted := []string{KindService, KindServiceImport, KindBackend, KindEdgeFunction}
89+
accepted := []string{KindService, KindServiceImport, KindBackend, KindEdgeFunction, KindVPCService}
9090
if backendRef.Kind != nil && !sets.NewString(accepted...).Has(string(*backendRef.Kind)) {
9191
parentRef.SetCondition(route,
9292
gwapiv1.RouteConditionResolvedRefs,

0 commit comments

Comments
 (0)