USHIFT-6796: C2CC: DNS forwarding between clusters

assets/components/openshift-dns/dns/configmap.yaml

@@ -1,6 +1,7 @@
 apiVersion: v1
 data:
   Corefile: |
+    {{- .C2CCDNSBlocks }}
     .:5353 {
         bufsize 1232
         errors

pkg/components/controllers.go

@@ -304,8 +304,12 @@ func startDNSController(ctx context.Context, cfg *config.Config, kubeconfigPath
 
 	hostsEnabled := cfg.DNS.Hosts.Status == config.HostsStatusEnabled
 	extraParams := assets.RenderParams{
-		"ClusterIP":    cfg.Network.DNS,
-		"HostsEnabled": hostsEnabled,
+		"ClusterIP":     cfg.Network.DNS,
+		"HostsEnabled":  hostsEnabled,
+		"C2CCDNSBlocks": "",
+	}
+	if cfg.C2CC.IsEnabled() {
+		extraParams["C2CCDNSBlocks"] = config.RenderC2CCDNSBlocks(cfg.C2CC.Resolved)
 	}
 
 	if err := assets.ApplyServices(ctx, svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {

pkg/config/c2cc.go

@@ -39,6 +39,7 @@ type ResolvedRemoteCluster struct {
 	ClusterNetwork []*net.IPNet
 	ServiceNetwork []*net.IPNet
 	Domain         string
+	DNSIP          string // 10th IP of ServiceNetwork[0], computed during validation when Domain is set
 }
 
 func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet {
@@ -266,6 +267,15 @@ func validateRemoteCluster(
 		}
 	}
 
+	if rc.Domain != "" && len(rc.ServiceNetwork) > 0 {
+		dnsIP, err := getClusterDNS(rc.ServiceNetwork[0])
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: failed to compute DNS IP from serviceNetwork[0] %q: %w", label, rc.ServiceNetwork[0], err))
+		} else {
+			res.DNSIP = dnsIP
+		}
+	}
+
 	errs = append(errs, validateIPFamilyConsistencyNets(res.ClusterNetwork, label+".clusterNetwork")...)
 	errs = append(errs, validateIPFamilyConsistencyNets(res.ServiceNetwork, label+".serviceNetwork")...)
 	errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...)
@@ -345,3 +355,33 @@ func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labe
 func cidrsOverlap(a, b *net.IPNet) bool {
 	return a.Contains(b.IP) || b.Contains(a.IP)
 }
+
+// RenderC2CCDNSBlocks generates CoreDNS server blocks for cross-cluster DNS.
+func RenderC2CCDNSBlocks(resolved []ResolvedRemoteCluster) string {
+	var blocks []string
+	for _, rc := range resolved {
+		if rc.Domain == "" {
+			continue
+		}
+		blocks = append(blocks, formatDNSBlock(rc.Domain, rc.DNSIP))
+	}
+	if len(blocks) == 0 {
+		return ""
+	}
+	return "\n" + strings.Join(blocks, "\n")
+}
+
+func formatDNSBlock(domain, dnsIP string) string {
+	return fmt.Sprintf(`    %s:5353 {
+        bufsize 1232
+        errors
+        log . {
+            class error
+        }
+        rewrite stop name suffix .%s .cluster.local answer auto
+        forward . %s
+        cache 10 {
+            denial 9984 10
+        }
+    }`, domain, domain, dnsIP)
+}

pkg/config/c2cc_test.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"net"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestC2CC_IsEnabled(t *testing.T) {
@@ -541,3 +543,113 @@ func TestC2CC_ValidateDualStack(t *testing.T) {
 		assert.NoError(t, cfg.C2CC.validate(cfg))
 	})
 }
+
+func TestC2CC_DNSIP(t *testing.T) {
+	stubHostIPs(t, nil)
+
+	t.Run("DNSIP populated when domain is set", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "10.100.0.2",
+				ClusterNetwork: []string{"10.45.0.0/16"},
+				ServiceNetwork: []string{"10.46.0.0/16"},
+				Domain:         "cluster-b.remote",
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, "10.46.0.10", cfg.C2CC.Resolved[0].DNSIP)
+	})
+
+	t.Run("DNSIP empty when domain is not set", func(t *testing.T) {
+		cfg := mkC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "10.100.0.2",
+				ClusterNetwork: []string{"10.45.0.0/16"},
+				ServiceNetwork: []string{"10.46.0.0/16"},
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Empty(t, cfg.C2CC.Resolved[0].DNSIP)
+	})
+
+	t.Run("DNSIP for IPv6 service network", func(t *testing.T) {
+		cfg := mkIPv6OnlyC2CCConfig(C2CC{
+			RemoteClusters: []RemoteCluster{{
+				NextHop:        "fd00::2",
+				ClusterNetwork: []string{"fd03::/48"},
+				ServiceNetwork: []string{"fd04::/112"},
+				Domain:         "cluster-b.remote",
+			}},
+		})
+		require.NoError(t, cfg.C2CC.validate(cfg))
+		assert.Equal(t, "fd04::a", cfg.C2CC.Resolved[0].DNSIP)
+	})
+}
+
+func parseCIDR(t *testing.T, s string) *net.IPNet {
+	t.Helper()
+	_, ipNet, err := net.ParseCIDR(s)
+	require.NoError(t, err)
+	return ipNet
+}
+
+func TestRenderC2CCDNSBlocks(t *testing.T) {
+	t.Run("no domains configured", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{{
+			NextHop:        net.ParseIP("10.100.0.2"),
+			ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")},
+			ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")},
+		}}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Empty(t, result)
+	})
+
+	t.Run("single domain", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{{
+			NextHop:        net.ParseIP("10.100.0.2"),
+			ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")},
+			ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")},
+			Domain:         "cluster-b.remote",
+			DNSIP:          "10.46.0.10",
+		}}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.True(t, strings.HasPrefix(result, "\n"), "result should start with newline for YAML block scalar")
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.Contains(t, result, "rewrite stop name suffix .cluster-b.remote .cluster.local answer auto")
+		assert.Contains(t, result, "forward . 10.46.0.10")
+		assert.Contains(t, result, "denial 9984 10")
+	})
+
+	t.Run("multiple domains", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{
+			{
+				Domain: "cluster-b.remote",
+				DNSIP:  "10.46.0.10",
+			},
+			{
+				Domain: "cluster-c.remote",
+				DNSIP:  "10.56.0.10",
+			},
+		}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.Contains(t, result, "forward . 10.46.0.10")
+		assert.Contains(t, result, "cluster-c.remote:5353")
+		assert.Contains(t, result, "forward . 10.56.0.10")
+	})
+
+	t.Run("mixed domain and no-domain", func(t *testing.T) {
+		resolved := []ResolvedRemoteCluster{
+			{
+				Domain: "cluster-b.remote",
+				DNSIP:  "10.46.0.10",
+			},
+			{
+				Domain: "",
+			},
+		}
+		result := RenderC2CCDNSBlocks(resolved)
+		assert.Contains(t, result, "cluster-b.remote:5353")
+		assert.NotContains(t, result, "cluster-c")
+	})
+}

pkg/controllers/c2cc/helpers_test.go

@@ -12,6 +12,7 @@ type testRemoteConfig struct {
 	nextHop        string
 	clusterNetwork []string
 	serviceNetwork []string
+	domain         string
 }
 
 func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRemoteConfig {
@@ -22,6 +23,15 @@ func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRem
 	}
 }
 
+func testRemoteWithDomain(nextHop string, clusterNetwork, serviceNetwork []string, domain string) testRemoteConfig {
+	return testRemoteConfig{
+		nextHop:        nextHop,
+		clusterNetwork: clusterNetwork,
+		serviceNetwork: serviceNetwork,
+		domain:         domain,
+	}
+}
+
 func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Config {
 	t.Helper()
 
@@ -32,6 +42,7 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co
 	for _, r := range remotes {
 		resolved := config.ResolvedRemoteCluster{
 			NextHop: net.ParseIP(r.nextHop),
+			Domain:  r.domain,
 		}
 		require.NotNil(t, resolved.NextHop, "invalid nextHop: %s", r.nextHop)
 

test/assets/c2cc/hello-microshift.yaml

@@ -10,10 +10,27 @@ spec:
   - name: hello-microshift
     image: quay.io/microshift/busybox:1.36
     command: ["/bin/sh"]
-    args: ["-c", "while true; do echo -ne \"HTTP/1.0 200 OK\r\nContent-Length: 16\r\n\r\nHello MicroShift\" | nc -l -p 8080 ; done"]
+    args:
+    - -c
+    - |
+      mkdir -p /tmp/www/cgi-bin
+      cat > /tmp/www/cgi-bin/hello <<SCRIPT
+      #!/bin/sh
+      echo "Content-Type: text/plain"
+      echo ""
+      SRC=\$(echo "\${REMOTE_ADDR}" | sed 's/\[//;s/\]//;s/^::ffff://')
+      echo "Hello from ${MY_POD_IP}, source: \${SRC}"
+      SCRIPT
+      chmod +x /tmp/www/cgi-bin/hello
+      httpd -f -p 8080 -h /tmp/www
     ports:
     - containerPort: 8080
       protocol: TCP
+    env:
+    - name: MY_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/resources/c2cc.resource

@@ -222,3 +222,28 @@ Remove Foreign Subnet From SNAT Annotation
     ...    Triggers a reconcile by briefly corrupting the annotation.
     [Arguments]    ${alias}
     Corrupt Node SNAT Annotation On Cluster    ${alias}
+
+Create Unique Namespace On Cluster
+    [Documentation]    Creates a namespace with a unique suffix on the given cluster.
+    ...    Returns the namespace name.
+    [Arguments]    ${alias}
+    ${rand}=    Generate Random String
+    ${rand}=    Convert To Lower Case    ${rand}
+    ${ns}=    Catenate    SEPARATOR=-    test    ${rand}
+    Oc On Cluster    ${alias}    oc create namespace ${ns}
+    RETURN    ${ns}
+
+Verify Corefile Contains C2CC Server Block
+    [Documentation]    Check that the CoreDNS Corefile configmap contains a server block for the given domain.
+    [Arguments]    ${alias}    ${domain}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get configmap dns-default -n openshift-dns -o jsonpath='{.data.Corefile}'
+    Should Contain    ${stdout}    ${domain}:5353
+    Should Contain    ${stdout}    rewrite stop name suffix .${domain} .cluster.local answer auto
+
+Verify Corefile Does Not Contain C2CC Server Block
+    [Documentation]    Check that the CoreDNS Corefile does NOT contain a server block for the given domain.
+    [Arguments]    ${alias}    ${domain}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get configmap dns-default -n openshift-dns -o jsonpath='{.data.Corefile}'
+    Should Not Contain    ${stdout}    ${domain}:5353

test/scenarios-bootc/el9/presubmits/el98-src@c2cc.sh

@@ -115,6 +115,7 @@ scenario_run_tests() {
         suites/c2cc/sanity.robot \
         suites/c2cc/infrastructure.robot \
         suites/c2cc/connectivity.robot \
+        suites/c2cc/dns.robot \
         suites/c2cc/reconciliation.robot \
         suites/c2cc/cleanup.robot
 }