USHIFT-6947: C2CC: Configurable routing table IDs

cmd/generate-config/config/config-openapi-spec.json

@@ -182,7 +182,8 @@
     "clusterToCluster": {
       "type": "object",
       "required": [
-        "dns"
+        "dns",
+        "routing"
       ],
       "properties": {
         "dns": {
@@ -243,6 +244,26 @@
               }
             }
           }
+        },
+        "routing": {
+          "description": "Linux policy routing table settings for C2CC routes.",
+          "type": "object",
+          "properties": {
+            "routeTableID": {
+              "description": "Linux policy routing table ID for direct routes to remote cluster CIDRs.\nThe route protocol number is set to the same value.\nMust be between 1 and 252 (253-255 are reserved by the kernel).\nMust differ from serviceRouteTableID.",
+              "type": "integer",
+              "default": 200,
+              "maximum": 252,
+              "minimum": 1
+            },
+            "serviceRouteTableID": {
+              "description": "Linux policy routing table ID for service routes via the OVN management port.\nThe route protocol number is set to the same value.\nMust be between 1 and 252 (253-255 are reserved by the kernel).\nMust differ from routeTableID.",
+              "type": "integer",
+              "default": 201,
+              "maximum": 252,
+              "minimum": 1
+            }
+          }
         }
       }
     },

docs/user/howto_config.md

@@ -40,6 +40,9 @@ clusterToCluster:
           domain: ""
           nextHop: ""
           serviceNetwork: []
+    routing:
+        routeTableID: 0
+        serviceRouteTableID: 0
 debugging:
     logLevel: ""
 dns:
@@ -206,6 +209,9 @@ clusterToCluster:
           domain: ""
           nextHop: ""
           serviceNetwork: []
+    routing:
+        routeTableID: 200
+        serviceRouteTableID: 201
 debugging:
     logLevel: Normal
 dns:

packaging/microshift/config.yaml

@@ -66,6 +66,18 @@ clusterToCluster:
           nextHop: ""
           # Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.
           serviceNetwork: []
+    # Linux policy routing table settings for C2CC routes.
+    routing:
+        # Linux policy routing table ID for direct routes to remote cluster CIDRs.
+        # The route protocol number is set to the same value.
+        # Must be between 1 and 252 (253-255 are reserved by the kernel).
+        # Must differ from serviceRouteTableID.
+        routeTableID: 200
+        # Linux policy routing table ID for service routes via the OVN management port.
+        # The route protocol number is set to the same value.
+        # Must be between 1 and 252 (253-255 are reserved by the kernel).
+        # Must differ from routeTableID.
+        serviceRouteTableID: 201
 debugging:
     # Valid values are: "Normal", "Debug", "Trace", "TraceAll".
     # Defaults to "Normal".

pkg/config/c2cc.go

@@ -26,9 +26,30 @@ type C2CCDNS struct {
 	CacheNegativeTTL *int `json:"cacheNegativeTTL,omitempty"`
 }
 
+type C2CCRouting struct {
+	// Linux policy routing table ID for direct routes to remote cluster CIDRs.
+	// The route protocol number is set to the same value.
+	// Must be between 1 and 252 (253-255 are reserved by the kernel).
+	// Must differ from serviceRouteTableID.
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=252
+	// +kubebuilder:default=200
+	RouteTableID *int `json:"routeTableID,omitempty"`
+	// Linux policy routing table ID for service routes via the OVN management port.
+	// The route protocol number is set to the same value.
+	// Must be between 1 and 252 (253-255 are reserved by the kernel).
+	// Must differ from routeTableID.
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=252
+	// +kubebuilder:default=201
+	ServiceRouteTableID *int `json:"serviceRouteTableID,omitempty"`
+}
+
 type C2CC struct {
 	// DNS cache settings for CoreDNS server blocks generated for remote clusters.
 	DNS C2CCDNS `json:"dns"`
+	// Linux policy routing table settings for C2CC routes.
+	Routing C2CCRouting `json:"routing"`
 	// List of remote clusters to establish connectivity with.
 	// C2CC is disabled when this list is empty.
 	RemoteClusters []RemoteCluster `json:"remoteClusters,omitempty"`
@@ -39,9 +60,11 @@ type C2CC struct {
 	ProbeInterval string `json:"probeInterval,omitempty"`
 
 	// Populated during validation with parsed network objects.
-	Resolved              []ResolvedRemoteCluster `json:"-"`
-	ResolvedAllCIDRs      []*net.IPNet            `json:"-"`
-	ResolvedProbeInterval time.Duration           `json:"-"`
+	Resolved                    []ResolvedRemoteCluster `json:"-"`
+	ResolvedAllCIDRs            []*net.IPNet            `json:"-"`
+	ResolvedProbeInterval       time.Duration           `json:"-"`
+	ResolvedRouteTableID        int                     `json:"-"`
+	ResolvedServiceRouteTableID int                     `json:"-"`
 }
 
 type RemoteCluster struct {
@@ -193,16 +216,58 @@ func parseAndValidateCIDR(cidr, field string, errs *[]error) *net.IPNet {
 	return ipNet
 }
 
+func (c *C2CC) resolveRoutingDefaults() {
+	routeTable := 200
+	if c.Routing.RouteTableID != nil {
+		routeTable = *c.Routing.RouteTableID
+	}
+	svcRouteTable := 201
+	if c.Routing.ServiceRouteTableID != nil {
+		svcRouteTable = *c.Routing.ServiceRouteTableID
+	}
+	c.ResolvedRouteTableID = routeTable
+	c.ResolvedServiceRouteTableID = svcRouteTable
+}
+
+func (c *C2CC) validateRouting() error {
+	c.resolveRoutingDefaults()
+
+	var errs []error
+	if c.ResolvedRouteTableID < 1 || c.ResolvedRouteTableID > 252 {
+		errs = append(errs, fmt.Errorf("routing.routeTableID must be between 1 and 252, got %d", c.ResolvedRouteTableID))
+	}
+	if c.ResolvedServiceRouteTableID < 1 || c.ResolvedServiceRouteTableID > 252 {
+		errs = append(errs, fmt.Errorf("routing.serviceRouteTableID must be between 1 and 252, got %d", c.ResolvedServiceRouteTableID))
+	}
+	if c.ResolvedRouteTableID == c.ResolvedServiceRouteTableID {
+		errs = append(errs, fmt.Errorf("routing.routeTableID (%d) and routing.serviceRouteTableID (%d) must differ",
+			c.ResolvedRouteTableID, c.ResolvedServiceRouteTableID))
+	}
+	return errors.Join(errs...)
+}
+
+func (d *C2CCDNS) validate() error {
+	var errs []error
+	if d.CacheTTL != nil && *d.CacheTTL < 0 {
+		errs = append(errs, fmt.Errorf("dns.cacheTTL must be >= 0, got %d", *d.CacheTTL))
+	}
+	if d.CacheNegativeTTL != nil && *d.CacheNegativeTTL < 0 {
+		errs = append(errs, fmt.Errorf("dns.cacheNegativeTTL must be >= 0, got %d", *d.CacheNegativeTTL))
+	}
+	return errors.Join(errs...)
+}
+
 func (c *C2CC) validate(cfg *Config) error {
 	if cfg.Network.CNIPlugin != CniPluginUnset && cfg.Network.CNIPlugin != CniPluginOVNK {
 		return fmt.Errorf("cluster to cluster requires OVN-Kubernetes CNI (network.cniPlugin must be \"\" or \"ovnk\", got %q)", cfg.Network.CNIPlugin)
 	}
 
-	if c.DNS.CacheTTL != nil && *c.DNS.CacheTTL < 0 {
-		return fmt.Errorf("dns.cacheTTL must be >= 0, got %d", *c.DNS.CacheTTL)
+	if err := c.DNS.validate(); err != nil {
+		return err
 	}
-	if c.DNS.CacheNegativeTTL != nil && *c.DNS.CacheNegativeTTL < 0 {
-		return fmt.Errorf("dns.cacheNegativeTTL must be >= 0, got %d", *c.DNS.CacheNegativeTTL)
+
+	if err := c.Routing.validate(c); err != nil {
+		return err
 	}
 
 	resolved, parseErrs := c.parseRemoteClusters()

pkg/config/c2cc_test.go

@@ -69,6 +69,16 @@ func withDNSDefaults(c2cc C2CC) C2CC {
 	return c2cc
 }
 
+func withRoutingDefaults(c2cc C2CC) C2CC {
+	if c2cc.Routing.RouteTableID == nil {
+		c2cc.Routing.RouteTableID = ptr.To(200)
+	}
+	if c2cc.Routing.ServiceRouteTableID == nil {
+		c2cc.Routing.ServiceRouteTableID = ptr.To(201)
+	}
+	return c2cc
+}
+
 func mkC2CCConfig(c2cc C2CC) *Config {
 	if c2cc.ProbeInterval == "" {
 		c2cc.ProbeInterval = "10s"
@@ -82,7 +92,7 @@ func mkC2CCConfig(c2cc C2CC) *Config {
 		Node: Node{
 			NodeIP: "10.100.0.1",
 		},
-		C2CC: withDNSDefaults(c2cc),
+		C2CC: withRoutingDefaults(withDNSDefaults(c2cc)),
 	}
 }
 
@@ -100,7 +110,7 @@ func mkDualStackC2CCConfig(c2cc C2CC) *Config {
 			NodeIP:   "10.100.0.1",
 			NodeIPV6: "fd00::1",
 		},
-		C2CC: withDNSDefaults(c2cc),
+		C2CC: withRoutingDefaults(withDNSDefaults(c2cc)),
 	}
 }
 
@@ -117,7 +127,7 @@ func mkIPv6OnlyC2CCConfig(c2cc C2CC) *Config {
 		Node: Node{
 			NodeIP: "fd00::1",
 		},
-		C2CC: withDNSDefaults(c2cc),
+		C2CC: withRoutingDefaults(withDNSDefaults(c2cc)),
 	}
 }
 
@@ -859,6 +869,82 @@ func TestRenderC2CCDNSBlocks(t *testing.T) {
 	})
 }
 
+func TestC2CC_RoutingTableValidation(t *testing.T) {
+	stubHostIPs(t, nil)
+
+	validRemote := []RemoteCluster{{
+		NextHop:        "10.100.0.2",
+		ClusterNetwork: []string{"10.45.0.0/16"},
+		ServiceNetwork: []string{"10.46.0.0/16"},
+	}}
+
+	t.Run("valid custom routing table IDs", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			Routing:        C2CCRouting{RouteTableID: ptr.To(100), ServiceRouteTableID: ptr.To(101)},
+			RemoteClusters: validRemote,
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, 100, cfg.C2CC.ResolvedRouteTableID)
+		assert.Equal(t, 101, cfg.C2CC.ResolvedServiceRouteTableID)
+	})
+
+	t.Run("boundary values 1 and 252", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			Routing:        C2CCRouting{RouteTableID: ptr.To(1), ServiceRouteTableID: ptr.To(252)},
+			RemoteClusters: validRemote,
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, 1, cfg.C2CC.ResolvedRouteTableID)
+		assert.Equal(t, 252, cfg.C2CC.ResolvedServiceRouteTableID)
+	})
+
+	t.Run("routeTableID below range", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			Routing:        C2CCRouting{RouteTableID: ptr.To(0), ServiceRouteTableID: ptr.To(201)},
+			RemoteClusters: validRemote,
+		})
+		err := cfg.C2CC.validate(cfg)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "routing.routeTableID must be between 1 and 252")
+	})
+
+	t.Run("serviceRouteTableID above range", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			Routing:        C2CCRouting{RouteTableID: ptr.To(200), ServiceRouteTableID: ptr.To(253)},
+			RemoteClusters: validRemote,
+		})
+		err := cfg.C2CC.validate(cfg)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "routing.serviceRouteTableID must be between 1 and 252")
+	})
+
+	t.Run("duplicate table IDs", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			Routing:        C2CCRouting{RouteTableID: ptr.To(150), ServiceRouteTableID: ptr.To(150)},
+			RemoteClusters: validRemote,
+		})
+		err := cfg.C2CC.validate(cfg)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "must differ")
+	})
+
+	t.Run("defaults are used when nil", func(t *testing.T) {
+		c2cc := C2CC{RemoteClusters: validRemote, ProbeInterval: "10s"}
+		cfg := &Config{
+			Network: Network{
+				CNIPlugin:      CniPluginOVNK,
+				ClusterNetwork: []string{"10.42.0.0/16"},
+				ServiceNetwork: []string{"10.43.0.0/16"},
+			},
+			Node: Node{NodeIP: "10.100.0.1"},
+			C2CC: withDNSDefaults(c2cc),
+		}
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, 200, cfg.C2CC.ResolvedRouteTableID)
+		assert.Equal(t, 201, cfg.C2CC.ResolvedServiceRouteTableID)
+	})
+}
+
 func TestC2CC_ProbeIntervalDefault(t *testing.T) {
 	cfg := &Config{}
 	require.NoError(t, cfg.fillDefaults())
@@ -879,6 +965,39 @@ func TestC2CC_IncorporateUserSettings(t *testing.T) {
 		assert.Equal(t, "30s", cfg.C2CC.ProbeInterval)
 	})
 
+	t.Run("user overrides routing table IDs", func(t *testing.T) {
+		cfg := &Config{}
+		require.NoError(t, cfg.fillDefaults())
+
+		user := &Config{
+			C2CC: C2CC{
+				Routing: C2CCRouting{
+					RouteTableID:        ptr.To(100),
+					ServiceRouteTableID: ptr.To(101),
+				},
+			},
+		}
+		cfg.incorporateUserSettings(user)
+		assert.Equal(t, 100, *cfg.C2CC.Routing.RouteTableID)
+		assert.Equal(t, 101, *cfg.C2CC.Routing.ServiceRouteTableID)
+	})
+
+	t.Run("user overrides only one routing table ID preserves other default", func(t *testing.T) {
+		cfg := &Config{}
+		require.NoError(t, cfg.fillDefaults())
+
+		user := &Config{
+			C2CC: C2CC{
+				Routing: C2CCRouting{
+					RouteTableID: ptr.To(100),
+				},
+			},
+		}
+		cfg.incorporateUserSettings(user)
+		assert.Equal(t, 100, *cfg.C2CC.Routing.RouteTableID)
+		assert.Equal(t, 201, *cfg.C2CC.Routing.ServiceRouteTableID)
+	})
+
 	t.Run("user sets remoteClusters without probeInterval preserves default", func(t *testing.T) {
 		cfg := &Config{}
 		require.NoError(t, cfg.fillDefaults())

pkg/config/config.go

@@ -201,6 +201,7 @@ func (c *Config) fillDefaults() error {
 	c.DNS = dnsDefaults()
 	c.C2CC = C2CC{
 		DNS:           C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(10)},
+		Routing:       C2CCRouting{RouteTableID: ptr.To(200), ServiceRouteTableID: ptr.To(201)},
 		ProbeInterval: "10s",
 	}
 	return nil
@@ -491,6 +492,12 @@ func (c *Config) incorporateUserSettings(u *Config) {
 	if u.C2CC.ProbeInterval != "" {
 		c.C2CC.ProbeInterval = u.C2CC.ProbeInterval
 	}
+	if u.C2CC.Routing.RouteTableID != nil {
+		c.C2CC.Routing.RouteTableID = u.C2CC.Routing.RouteTableID
+	}
+	if u.C2CC.Routing.ServiceRouteTableID != nil {
+		c.C2CC.Routing.ServiceRouteTableID = u.C2CC.Routing.ServiceRouteTableID
+	}
 }
 
 // updateComputedValues examins the existing settings and converts any
@@ -578,6 +585,7 @@ func (c *Config) updateComputedValues() error {
 	}
 
 	c.C2CC.stripEmptyRemoteClusters()
+	c.C2CC.resolveRoutingDefaults()
 
 	return nil
 }

pkg/controllers/c2cc/controller.go

@@ -83,13 +83,13 @@ func (c *C2CCRouteManager) Run(ctx context.Context, ready chan<- struct{}, stopp
 	c.ovn.subscribe(ctx, reconcileCh)
 
 	if routeDone, err := c.routes.subscribe(reconcileCh, "linux-route-change"); err != nil {
-		klog.Warningf("Could not subscribe to route events for table %d: %v", c2ccRouteTable, err)
+		klog.Warningf("Could not subscribe to route events for table %d: %v", c.routes.table, err)
 	} else {
 		defer close(routeDone)
 	}
 
 	if svcRouteDone, err := c.svcRoutes.subscribe(reconcileCh, "service-route-change"); err != nil {
-		klog.Warningf("Could not subscribe to route events for table %d: %v", c2ccSvcRouteTable, err)
+		klog.Warningf("Could not subscribe to route events for table %d: %v", c.svcRoutes.table, err)
 	} else {
 		defer close(svcRouteDone)
 	}