feat: add support for CoreDNS forwarding to host DNS

This PR adds the support for CoreDNS forwarding to host DNS. We try to bind on 9th address on the first element from `serviceSubnets` and create a simple service so k8s will not attempt to rebind it. Signed-off-by: Andrey Smirnov <[email protected]> Co-authored-by: Dmitriy Matrenichev <[email protected]> Signed-off-by: Dmitriy Matrenichev <[email protected]>
siderolabs · Apr 3, 2024 · 862c760 · 862c760
1 parent e8ae5ef
commit 862c760
Show file tree

Hide file tree

Showing 41 changed files with 2,581 additions and 1,463 deletions.
diff --git a/api/resource/definitions/k8s/k8s.proto b/api/resource/definitions/k8s/k8s.proto
@@ -60,6 +60,7 @@ message BootstrapManifestsConfigSpec {
   repeated string flannel_extra_args = 16;
   string flannel_kube_service_host = 17;
   string flannel_kube_service_port = 18;
+  string service_host_dns_address = 19;
 }
 
 // ConfigStatusSpec describes status of rendered secrets.

diff --git a/api/resource/definitions/network/network.proto b/api/resource/definitions/network/network.proto
@@ -101,6 +101,13 @@ message HardwareAddrSpec {
   bytes hardware_addr = 2;
 }
 
+// HostDNSConfigSpec describes host DNS config.
+message HostDNSConfigSpec {
+  bool enabled = 1;
+  repeated common.NetIPPort listen_addresses = 2;
+  common.NetIP service_host_dns_address = 3;
+}
+
 // HostnameSpecSpec describes node hostname.
 message HostnameSpecSpec {
   string hostname = 1;

diff --git a/go.mod b/go.mod
@@ -92,8 +92,8 @@ require (
 	github.com/jeromer/syslogparser v1.1.0
 	github.com/jsimonetti/rtnetlink v1.4.1
 	github.com/jxskiss/base62 v1.1.0
-	github.com/linode/go-metadata v0.2.0
 	github.com/klauspost/cpuid/v2 v2.2.7
+	github.com/linode/go-metadata v0.2.0
 	github.com/martinlindhe/base36 v1.1.1
 	github.com/mattn/go-isatty v0.0.20
 	github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
@@ -177,8 +177,6 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
-require gopkg.in/yaml.v2 v2.4.0
-
 require (
 	cloud.google.com/go/compute v1.23.4 // indirect
 	github.com/0x5a17ed/itkit v0.6.0 // indirect
@@ -354,6 +352,7 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/cli-runtime v0.30.0-rc.1 // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane.go b/internal/app/machined/pkg/controllers/k8s/control_plane.go
@@ -13,7 +13,10 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/controller/generic"
 	"github.com/cosi-project/runtime/pkg/controller/generic/transform"
+	"github.com/cosi-project/runtime/pkg/safe"
+	"github.com/cosi-project/runtime/pkg/state"
 	"github.com/siderolabs/gen/optional"
+	"github.com/siderolabs/gen/value"
 	"github.com/siderolabs/gen/xslices"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
@@ -25,6 +28,7 @@ import (
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
 	"github.com/siderolabs/talos/pkg/machinery/resources/config"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
+	"github.com/siderolabs/talos/pkg/machinery/resources/network"
 )
 
 // controlplaneMapFunc is a shared "map" func for transform controller which guards on:
@@ -209,6 +213,8 @@ func NewControlPlaneSchedulerController() *ControlPlaneSchedulerController {
 type ControlPlaneBootstrapManifestsController = transform.Controller[*config.MachineConfig, *k8s.BootstrapManifestsConfig]
 
 // NewControlPlaneBootstrapManifestsController instanciates the controller.
+//
+//nolint:gocyclo
 func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifestsController {
 	return transform.NewController(
 		transform.Settings[*config.MachineConfig, *k8s.BootstrapManifestsConfig]{
@@ -255,6 +261,17 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					server = cfgProvider.Cluster().Endpoint().String()
 				}
 
+				hostDNSCfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID)
+				if err != nil && !state.IsNotFoundError(err) {
+					return fmt.Errorf("error getting host DNS config: %w", err)
+				}
+
+				var serviceHostDNSAddress string
+
+				if hostDNSCfg != nil && !value.IsZero(hostDNSCfg.TypedSpec().ServiceHostDNSAddress) {
+					serviceHostDNSAddress = hostDNSCfg.TypedSpec().ServiceHostDNSAddress.String()
+				}
+
 				*res.TypedSpec() = k8s.BootstrapManifestsConfigSpec{
 					Server:        server,
 					ClusterDomain: cfgProvider.Cluster().Network().DNSDomain(),
@@ -281,11 +298,21 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					PodSecurityPolicyEnabled: !cfgProvider.Cluster().APIServer().DisablePodSecurityPolicy(),
 
 					TalosAPIServiceEnabled: cfgProvider.Machine().Features().KubernetesTalosAPIAccess().Enabled(),
+
+					ServiceHostDNSAddress: serviceHostDNSAddress,
 				}
 
 				return nil
 			},
 		},
+		transform.WithExtraInputs(
+			controller.Input{
+				Namespace: network.NamespaceName,
+				Type:      network.HostDNSConfigType,
+				ID:        optional.Some(network.HostDNSConfigID),
+				Kind:      controller.InputWeak,
+			},
+		),
 	)
 }
 

diff --git a/internal/app/machined/pkg/controllers/k8s/manifest.go b/internal/app/machined/pkg/controllers/k8s/manifest.go
@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strings"
 	"text/template"
 
@@ -168,6 +169,8 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
 		ApidPort int
 
 		TalosServiceAccount TalosServiceAccount
+
+		HostDNSAddr string
 	}{
 		BootstrapManifestsConfigSpec: cfg,
 		Secrets:                      scrt,
@@ -201,44 +204,42 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
 	}
 
 	if cfg.CoreDNSEnabled {
-		defaultManifests = append(defaultManifests,
+		defaultManifests = slices.Concat(defaultManifests,
 			[]manifestDesc{
 				{"11-core-dns", coreDNSTemplate},
 				{"11-core-dns-svc", coreDNSSvcTemplate},
-			}...,
+			},
 		)
 	}
 
 	if cfg.FlannelEnabled {
 		defaultManifests = append(defaultManifests,
-			[]manifestDesc{
-				{"05-flannel", flannelTemplate},
-			}...,
-		)
+			manifestDesc{"05-flannel", flannelTemplate})
 	}
 
 	if cfg.ProxyEnabled {
 		defaultManifests = append(defaultManifests,
-			[]manifestDesc{
-				{"10-kube-proxy", kubeProxyTemplate},
-			}...,
-		)
+			manifestDesc{"10-kube-proxy", kubeProxyTemplate})
 	}
 
 	if cfg.PodSecurityPolicyEnabled {
 		defaultManifests = append(defaultManifests,
-			[]manifestDesc{
-				{"03-default-pod-security-policy", podSecurityPolicy},
-			}...,
+			manifestDesc{"03-default-pod-security-policy", podSecurityPolicy},
 		)
 	}
 
 	if cfg.TalosAPIServiceEnabled {
-		defaultManifests = append(defaultManifests,
+		defaultManifests = slices.Concat(defaultManifests,
 			[]manifestDesc{
 				{"12-talos-api-service", talosAPIService},
 				{"13-talos-service-account-crd", talosServiceAccountCRDTemplate},
-			}...,
+			},
+		)
+	}
+
+	if cfg.ServiceHostDNSAddress != "" {
+		defaultManifests = append(defaultManifests,
+			manifestDesc{"15-host-dns-service", talosHostDNSSvcTemplate},
 		)
 	}
 

diff --git a/internal/app/machined/pkg/controllers/k8s/templates.go b/internal/app/machined/pkg/controllers/k8s/templates.go
@@ -571,3 +571,22 @@ spec:
       served: true
       storage: true
 `)
+
+var talosHostDNSSvcTemplate = []byte(`apiVersion: v1
+kind: Service
+metadata:
+  name: host-dns
+  namespace: kube-system
+spec:
+  clusterIP: {{ .ServiceHostDNSAddress }}
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+    targetPort: 53
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+    targetPort: 53
+  type: ClusterIP
+`)