Merge pull request #30946 from gcs278/enable-gateway-api-vsphere

NE-2520: Enable Gateway API tests on vSphere and baremetal

Author: openshift-merge-bot[bot]

test/extended/router/gatewayapicontroller.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 	admissionapi "k8s.io/pod-security-admission/api"
+	utilnet "k8s.io/utils/net"
 	"k8s.io/utils/pointer"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -104,11 +105,13 @@ var (
 var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feature:Router][apigroup:gateway.networking.k8s.io]", g.Ordered, g.Serial, func() {
 	defer g.GinkgoRecover()
 	var (
-		oc         = exutil.NewCLIWithPodSecurityLevel("gatewayapi-controller", admissionapi.LevelBaseline)
-		csvName    string
-		err        error
-		gateways   []string
-		infPoolCRD = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/main/config/crd/bases/inference.networking.k8s.io_inferencepools.yaml"
+		oc                    = exutil.NewCLIWithPodSecurityLevel("gatewayapi-controller", admissionapi.LevelBaseline)
+		csvName               string
+		err                   error
+		gateways              []string
+		infPoolCRD            = "https://raw.githubusercontent.com/kubernetes-sigs/gateway-api-inference-extension/main/config/crd/bases/inference.networking.k8s.io_inferencepools.yaml"
+		managedDNS            bool
+		loadBalancerSupported bool
 	)
 
 	const (
@@ -127,7 +130,9 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 			g.Skip("Skipping on OKD cluster as OSSM is not available as a community operator")
 		}
 
-		skipGatewayForUnsupportedPlatform(oc)
+		// Check platform support and get capabilities (LoadBalancer, DNS)
+		loadBalancerSupported, managedDNS = checkPlatformSupportAndGetCapabilities(oc)
+
 		if !isNoOLMFeatureGateEnabled(oc) {
 			// GatewayAPIController without GatewayAPIWithoutOLM featuregate
 			// relies on OSSM OLM operator.
@@ -307,7 +312,10 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		g.By("Check OLM catalogSource, subscription, CSV and Pod")
 		waitCatalogErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 20*time.Minute, false, func(context context.Context) (bool, error) {
 			catalog, err := oc.AsAdmin().Run("get").Args("-n", "openshift-marketplace", "catalogsource", expectedSubscriptionSource, "-o=jsonpath={.status.connectionState.lastObservedState}").Output()
-			o.Expect(err).NotTo(o.HaveOccurred())
+			if err != nil {
+				e2e.Logf("Failed to get CatalogSource %q: %v; retrying...", expectedSubscriptionSource, err)
+				return false, nil
+			}
 			if catalog != "READY" {
 				e2e.Logf("CatalogSource %q is not in ready state, retrying...", expectedSubscriptionSource)
 				return false, nil
@@ -335,7 +343,10 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 
 		waitCSVErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 20*time.Minute, false, func(context context.Context) (bool, error) {
 			csvStatus, err := oc.AsAdmin().Run("get").Args("-n", expectedSubscriptionNamespace, "clusterserviceversion", csvName, "-o=jsonpath={.status.phase}").Output()
-			o.Expect(err).NotTo(o.HaveOccurred())
+			if err != nil {
+				e2e.Logf("Failed to get ClusterServiceVersion %q: %v; retrying...", csvName, err)
+				return false, nil
+			}
 			if csvStatus != "Succeeded" {
 				e2e.Logf("Cluster Service Version %q is not successful, retrying...", csvName)
 				return false, nil
@@ -440,14 +451,18 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		g.By("Create the default Gateway")
 		gw := names.SimpleNameGenerator.GenerateName("gateway-")
 		gateways = append(gateways, gw)
-		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, defaultDomain)
+		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, defaultDomain, loadBalancerSupported)
 		o.Expect(gwerr).NotTo(o.HaveOccurred(), "failed to create Gateway")
 
 		g.By("Verify the gateway's LoadBalancer service and DNSRecords")
-		assertGatewayLoadbalancerReady(oc, gw, gw+"-openshift-default")
+		if loadBalancerSupported {
+			assertGatewayLoadbalancerReady(oc, gw, gw+"-openshift-default")
+		}
 
 		// check the dns record is created and status of the published dnsrecord of all zones are True
-		assertDNSRecordStatus(oc, gw)
+		if managedDNS {
+			assertDNSRecordStatus(oc, gw)
+		}
 	})
 
 	g.It("Ensure HTTPRoute object is created", func() {
@@ -465,11 +480,13 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		g.By("Create a custom Gateway for the HTTPRoute")
 		gw := names.SimpleNameGenerator.GenerateName("gateway-")
 		gateways = append(gateways, gw)
-		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, customDomain)
+		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, customDomain, loadBalancerSupported)
 		o.Expect(gwerr).NotTo(o.HaveOccurred(), "Failed to create Gateway")
 
 		// make sure the DNSRecord is ready to use
-		assertDNSRecordStatus(oc, gw)
+		if managedDNS {
+			assertDNSRecordStatus(oc, gw)
+		}
 
 		g.By("Create the http route using the custom gateway")
 		defaultRoutename := "test-hostname." + customDomain
@@ -479,7 +496,9 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		assertHttpRouteSuccessful(oc, gw, "test-httproute")
 
 		g.By("Validating the http connectivity to the backend application")
-		assertHttpRouteConnection(defaultRoutename)
+		if loadBalancerSupported && managedDNS {
+			assertHttpRouteConnection(defaultRoutename)
+		}
 	})
 
 	g.It("Ensure GIE is enabled after creating an inferencePool CRD", func() {
@@ -565,6 +584,10 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 	})
 
 	g.It("Ensure gateway loadbalancer service and dnsrecords could be deleted and then get recreated [Serial]", func() {
+		if !loadBalancerSupported || !managedDNS {
+			g.Skip("Skipping LoadBalancer and DNS deletion test - platform does not support these features")
+		}
+
 		g.By("Getting the default domain for creating a custom Gateway")
 		defaultIngressDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute)
 		o.Expect(err).NotTo(o.HaveOccurred(), "Failed to find default domain name")
@@ -573,7 +596,7 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 		g.By("Create a custom Gateway")
 		gw := names.SimpleNameGenerator.GenerateName("gateway-")
 		gateways = append(gateways, gw)
-		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, customDomain)
+		_, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, customDomain, loadBalancerSupported)
 		o.Expect(gwerr).NotTo(o.HaveOccurred(), "Failed to create Gateway")
 
 		// verify the gateway's LoadBalancer service
@@ -610,10 +633,9 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat
 	})
 })
 
-// skipGatewayForUnsupportedPlatform skips gateway API tests on non-cloud
-// platforms (gateway needs LB service) and on dual-stack clusters (dual-stack
-// support is not yet declared).
-func skipGatewayForUnsupportedPlatform(oc *exutil.CLI) {
+// checkPlatformSupportAndGetCapabilities verifies the platform is supported and returns
+// platform capabilities for LoadBalancer services and managed DNS.
+func checkPlatformSupportAndGetCapabilities(oc *exutil.CLI) (loadBalancerSupported bool, managedDNS bool) {
 	infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	o.Expect(infra).NotTo(o.BeNil())
@@ -623,17 +645,47 @@ func skipGatewayForUnsupportedPlatform(oc *exutil.CLI) {
 	o.Expect(platformType).NotTo(o.BeEmpty())
 	switch platformType {
 	case configv1.AWSPlatformType, configv1.AzurePlatformType, configv1.GCPPlatformType, configv1.IBMCloudPlatformType:
-		// supported
+		// Cloud platforms with native LoadBalancer support
+		loadBalancerSupported = true
+	case configv1.VSpherePlatformType, configv1.BareMetalPlatformType, configv1.EquinixMetalPlatformType:
+		// Platforms without native LoadBalancer support (may have MetalLB or similar)
+		loadBalancerSupported = false
 	default:
-		g.Skip(fmt.Sprintf("Skipping on non cloud platform type %q", platformType))
+		g.Skip(fmt.Sprintf("Skipping on unsupported platform type %q", platformType))
 	}
 
-	if infra.Status.PlatformStatus.AWS != nil {
-		ipFamily := infra.Status.PlatformStatus.AWS.IPFamily
-		if ipFamily == configv1.DualStackIPv4Primary || ipFamily == configv1.DualStackIPv6Primary {
-			g.Skip("Skipping Gateway API tests on dual-stack cluster")
+	// Check if DNS is managed (has public or private zones configured)
+	managedDNS = isDNSManaged(oc)
+
+	// Skip Gateway API tests on IPv6 or dual-stack clusters (any platform)
+	if isIPv6OrDualStack(oc) {
+		g.Skip("Skipping Gateway API tests on IPv6/dual-stack cluster")
+	}
+
+	e2e.Logf("Platform: %s, LoadBalancer supported: %t, DNS managed: %t", platformType, loadBalancerSupported, managedDNS)
+	return loadBalancerSupported, managedDNS
+}
+
+// isDNSManaged checks if the cluster has DNS zones configured (public or private).
+// On platforms like vSphere without external DNS, DNS records cannot be managed.
+func isDNSManaged(oc *exutil.CLI) bool {
+	dnsConfig, err := oc.AdminConfigClient().ConfigV1().DNSes().Get(context.Background(), "cluster", metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get DNS config")
+	return dnsConfig.Spec.PrivateZone != nil || dnsConfig.Spec.PublicZone != nil
+}
+
+// isIPv6OrDualStack checks if the cluster is using IPv6 or dual-stack networking.
+// Returns true if any ServiceNetwork CIDR is IPv6 (indicates IPv6-only or dual-stack).
+func isIPv6OrDualStack(oc *exutil.CLI) bool {
+	networkConfig, err := oc.AdminOperatorClient().OperatorV1().Networks().Get(context.Background(), "cluster", metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred(), "Failed to get network config")
+
+	for _, cidr := range networkConfig.Spec.ServiceNetwork {
+		if utilnet.IsIPv6CIDRString(cidr) {
+			return true
 		}
 	}
+	return false
 }
 
 func isNoOLMFeatureGateEnabled(oc *exutil.CLI) bool {
@@ -679,7 +731,7 @@ func buildGatewayClass(name, controllerName string) *gatewayapiv1.GatewayClass {
 }
 
 // createAndCheckGateway build and creates the Gateway.
-func createAndCheckGateway(oc *exutil.CLI, gwname, gwclassname, domain string) (*gatewayapiv1.Gateway, error) {
+func createAndCheckGateway(oc *exutil.CLI, gwname, gwclassname, domain string, loadBalancerSupported bool) (*gatewayapiv1.Gateway, error) {
 	// Build the gateway object
 	gatewaybuild := buildGateway(gwname, ingressNamespace, gwclassname, "All", domain)
 
@@ -690,10 +742,19 @@ func createAndCheckGateway(oc *exutil.CLI, gwname, gwclassname, domain string) (
 	}
 
 	// Confirm the gateway is up and running
-	return checkGatewayStatus(oc, gwname, ingressNamespace)
+	return checkGatewayStatus(oc, gwname, ingressNamespace, loadBalancerSupported)
 }
 
-func checkGatewayStatus(oc *exutil.CLI, gwname, ingressNameSpace string) (*gatewayapiv1.Gateway, error) {
+func checkGatewayStatus(oc *exutil.CLI, gwname, ingressNameSpace string, loadBalancerSupported bool) (*gatewayapiv1.Gateway, error) {
+	// Determine which condition to wait for based on platform capabilities
+	// Without LoadBalancer support, Gateway reaches Accepted but not Programmed (reason: AddressNotAssigned)
+	var expectedCondition gatewayapiv1.GatewayConditionType
+	if loadBalancerSupported {
+		expectedCondition = gatewayapiv1.GatewayConditionProgrammed
+	} else {
+		expectedCondition = gatewayapiv1.GatewayConditionAccepted
+	}
+
 	programmedGateway := &gatewayapiv1.Gateway{}
 	timeout := 20 * time.Minute
 	if err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, timeout, false, func(context context.Context) (bool, error) {
@@ -702,22 +763,21 @@ func checkGatewayStatus(oc *exutil.CLI, gwname, ingressNameSpace string) (*gatew
 			e2e.Logf("Failed to get gateway %q: %v, retrying...", gwname, err)
 			return false, nil
 		}
-		// Checking the gateway controller status
 		for _, condition := range gateway.Status.Conditions {
-			if condition.Type == string(gatewayapiv1.GatewayConditionProgrammed) {
+			if condition.Type == string(expectedCondition) {
 				if condition.Status == metav1.ConditionTrue {
-					e2e.Logf("The gateway controller for gateway %q is programmed", gwname)
+					e2e.Logf("Gateway %q has condition %s=True", gwname, expectedCondition)
 					programmedGateway = gateway
 					return true, nil
 				}
 			}
 		}
-		e2e.Logf("Found gateway %q but the controller is still not programmed, retrying...", gwname)
+		e2e.Logf("Found gateway %q but condition %s is not yet True, retrying...", gwname, expectedCondition)
 		return false, nil
 	}); err != nil {
-		return nil, fmt.Errorf("timed out after %v waiting for gateway %q to become programmed: %w", timeout, gwname, err)
+		return nil, fmt.Errorf("timed out after %v waiting for gateway %q to have condition %s=True: %w", timeout, gwname, expectedCondition, err)
 	}
-	e2e.Logf("Gateway %q successfully programmed!", gwname)
+	e2e.Logf("Gateway %q successfully has condition %s=True", gwname, expectedCondition)
 	return programmedGateway, nil
 }
 
@@ -843,7 +903,10 @@ func createHttpRoute(oc *exutil.CLI, gwName, routeName, hostname, backendRefname
 	// Confirm the HTTPRoute is up
 	waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 4*time.Minute, false, func(context context.Context) (bool, error) {
 		checkHttpRoute, err := oc.GatewayApiClient().GatewayV1().HTTPRoutes(namespace).Get(context, httpRoute.Name, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred())
+		if err != nil {
+			e2e.Logf("Failed to get HTTPRoute %q: %v; retrying...", httpRoute.Name, err)
+			return false, nil
+		}
 		if len(checkHttpRoute.Status.Parents) > 0 {
 			for _, condition := range checkHttpRoute.Status.Parents[0].Conditions {
 				if condition.Type == string(gatewayapiv1.RouteConditionAccepted) {
@@ -965,7 +1028,10 @@ func assertHttpRouteSuccessful(oc *exutil.CLI, gwName, name string) (*gatewayapi
 	// Wait up to 4 minutes for parent(s) to update.
 	err := wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 4*time.Minute, false, func(context context.Context) (bool, error) {
 		checkHttpRoute, err := oc.GatewayApiClient().GatewayV1().HTTPRoutes(namespace).Get(context, name, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred())
+		if err != nil {
+			e2e.Logf("Failed to get HTTPRoute %s/%s: %v; retrying...", namespace, name, err)
+			return false, nil
+		}
 
 		numParents := len(checkHttpRoute.Status.Parents)
 		if numParents == 0 {