aenix-io · kvaps · Jul 21, 2021 · Jul 20, 2021
diff --git a/deploy/helm/kubernetes/manifests/konnectivity-agent-deployment.yaml b/deploy/helm/kubernetes/manifests/konnectivity-agent-deployment.yaml
@@ -15,7 +15,11 @@ metadata:
   namespace: kube-system
   name: konnectivity-agent
 spec:
+  {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
   replicas: {{ .Values.konnectivityAgent.replicaCount }}
+  {{- else }}
+  replicas: {{ .Values.apiServer.replicaCount }}
+  {{- end }}
   selector:
     matchLabels:
       k8s-app: konnectivity-agent
@@ -83,8 +87,11 @@ spec:
         - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token
 
         {{- if not (hasKey .Values.konnectivityAgent.extraArgs "proxy-server-host") }}
-        {{- if .Values.konnectivityServer.service.loadBalancerIP }}
+        {{- if and (eq .Values.konnectivityServer.mode "HTTPConnect") .Values.konnectivityServer.service.loadBalancerIP }}
         - --proxy-server-host={{ .Values.konnectivityServer.service.loadBalancerIP }}
+        {{- else if .Values.konnectivityServer.service.NodePort }}
+        {{- else if and (eq .Values.konnectivityServer.mode "GRPC") .Values.apiServer.service.loadBalancerIP }}
+        - --proxy-server-host={{ .Values.apiServer.service.loadBalancerIP }}
         {{- else }}
         {{- fail ".konnectivityAgent.extraArgs.proxy-server-host must be specified!" }}
         {{- end }}

diff --git a/deploy/helm/kubernetes/templates/_helpers.tpl b/deploy/helm/kubernetes/templates/_helpers.tpl
@@ -74,3 +74,115 @@ Take the first IP address from the serviceClusterIPRange for the kube-dns servic
   {{- $octetsList := splitList "." .Values.apiServer.serviceClusterIPRange -}}
   {{- printf "%d.%d.%d.%d" (index $octetsList 0 | int) (index $octetsList 1 | int) (index $octetsList 2 | int) 1 -}}
 {{- end -}}
+
+{{/*
+Template for konnectivityServer containers
+*/}}
+{{- define "kubernetes.konnectivityServer.containers" -}}
+      - command:
+        - /proxy-server
+        - --logtostderr=true
+        - --server-count={{ .Values.konnectivityServer.replicaCount }}
+        - --server-id=$(POD_NAME)
+        - --cluster-cert=/pki/apiserver/tls.crt
+        - --cluster-key=/pki/apiserver/tls.key
+        {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
+        - --mode=http-connect
+        - --server-port={{ .Values.konnectivityServer.ports.server }}
+        - --server-ca-cert=/pki/konnectivity-server/ca.crt
+        - --server-cert=/pki/konnectivity-server/tls.crt
+        - --server-key=/pki/konnectivity-server/tls.key
+        {{- else }}
+        - --mode=grpc
+        - --uds-name=/run/konnectivity-server/konnectivity-server.socket
+        - --server-port=0
+        {{- end }}
+        - --agent-port={{ .Values.konnectivityServer.ports.agent }}
+        - --admin-port={{ .Values.konnectivityServer.ports.admin }}
+        - --health-port={{ .Values.konnectivityServer.ports.health }}
+        - --agent-namespace=kube-system
+        - --agent-service-account=konnectivity-agent
+        - --kubeconfig=/etc/kubernetes/konnectivity-server.conf
+        - --authentication-audience=system:konnectivity-server
+        {{- range $key, $value := .Values.konnectivityServer.extraArgs }}
+        - --{{ $key }}={{ $value }}
+        {{- end }}
+        ports:
+        {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
+        - containerPort: {{ .Values.konnectivityServer.ports.server }}
+          name: server
+        {{- end }}
+        - containerPort: {{ .Values.konnectivityServer.ports.agent }}
+          name: agent
+        - containerPort: {{ .Values.konnectivityServer.ports.admin }}
+          name: admin
+        - containerPort: {{ .Values.konnectivityServer.ports.health }}
+          name: health
+        {{- with .Values.konnectivityServer.image }}
+        image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}"
+        imagePullPolicy: {{ .pullPolicy }}
+        {{- end }}
+        livenessProbe:
+          failureThreshold: 8
+          httpGet:
+            path: /healthz
+            port: {{ .Values.konnectivityServer.ports.health }}
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 60
+        name: konnectivity-server
+        resources:
+          {{- toYaml .Values.konnectivityServer.resources | nindent 10 }}
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        {{- with .Values.konnectivityServer.extraEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        volumeMounts:
+        - mountPath: /pki/apiserver
+          name: pki-apiserver
+        {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
+        - mountPath: /pki/konnectivity-server
+          name: pki-konnectivity-server
+        {{- else }}
+        - mountPath: /run/konnectivity-server
+          name: konnectivity-uds
+        {{- end }}
+        - mountPath: /pki/konnectivity-server-client
+          name: pki-konnectivity-server-client
+        - mountPath: /etc/kubernetes/
+          name: kubeconfig
+          readOnly: true
+        {{- with .Values.konnectivityServer.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+{{- end -}}
+
+{{/*
+Template for konnectivityServer volumes
+*/}}
+{{- define "kubernetes.konnectivityServer.volumes" -}}
+      - secret:
+          secretName: "{{ template "kubernetes.fullname" . }}-pki-apiserver-server"
+        name: pki-apiserver
+      {{- if eq .Values.konnectivityServer.mode "HTTPConnect" }}
+      - secret:
+          secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server"
+        name: pki-konnectivity-server
+      {{- else }}
+      - secret:
+          secretName: "{{ template "kubernetes.fullname" . }}-pki-konnectivity-server-client"
+        name: pki-konnectivity-server-client
+      - emptyDir: {}
+        name: konnectivity-uds
+      {{- end }}
+      - configMap:
+          name: "{{ template "kubernetes.fullname" . }}-konnectivity-server-conf"
+        name: kubeconfig
+      {{- with .Values.konnectivityServer.extraVolumes }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+{{- end -}}
diff --git a/deploy/helm/kubernetes/templates/apiserver-config.yaml b/deploy/helm/kubernetes/templates/apiserver-config.yaml
@@ -12,14 +12,23 @@ data:
     - name: cluster
       connection:
         {{- if and .Values.konnectivityServer.enabled }}
-        proxyProtocol: HTTPConnect
+        {{- if has .Values.konnectivityServer.mode (list "HTTPConnect" "GRPC") }}
+        proxyProtocol: {{ .Values.konnectivityServer.mode }}
+        {{- else }}
+        {{- fail ".Values.konnectivityServer.mode supports only \"HTTPConnect\" and \"GRPC\" values" }}
+        {{- end }}
         transport:
+          {{- if eq .Values.konnectivityServer.mode "GRPC" }}
+          uds:
+            udsName: /run/konnectivity-server/konnectivity-server.socket
+          {{- else }}
           tcp:
             url: "https://{{ $fullName }}-konnectivity-server:8131"
             TLSConfig:
               caBundle: /pki/konnectivity-client/ca.crt
               clientKey: /pki/konnectivity-client/tls.key
               clientCert: /pki/konnectivity-client/tls.crt
+          {{- end }}
         {{- else }}
         proxyProtocol: Direct
         {{- end }}

diff --git a/deploy/helm/kubernetes/templates/apiserver-deployment.yaml b/deploy/helm/kubernetes/templates/apiserver-deployment.yaml
@@ -101,7 +101,7 @@ spec:
         - --egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml
         - --service-account-issuer=https://kubernetes.default.svc.cluster.local
         - --service-account-signing-key-file=/pki/sa/tls.key
-        {{- if .Values.konnectivityAgent.enabled }}{{"\n"}}
+        {{- if .Values.konnectivityAgent.enabled }}
         - --api-audiences=system:konnectivity-server
         {{- end }}
         {{- if not (hasKey .Values.apiServer.extraArgs "advertise-address") }}
@@ -147,13 +147,20 @@ spec:
           name: pki-apiserver-kubelet-client
         - mountPath: /pki/sa
           name: pki-sa
-        {{- if .Values.konnectivityServer.enabled }}{{"\n"}}
+        {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
         - mountPath: /pki/konnectivity-client
           name: pki-konnectivity-client
         {{- end }}
+        {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
+        - mountPath: /run/konnectivity-server
+          name: konnectivity-uds
+        {{- end }}
         {{- with .Values.apiServer.extraVolumeMounts }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+      {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
+      {{ template "kubernetes.konnectivityServer.containers" . }}
+      {{- end }}
       {{- with .Values.apiServer.sidecars }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
@@ -176,11 +183,14 @@ spec:
       - secret:
           secretName: "{{ $fullName }}-pki-sa"
         name: pki-sa
-      {{- if .Values.konnectivityServer.enabled }}{{"\n"}}
+      {{- if .Values.konnectivityServer.enabled }}
       - secret:
           secretName: "{{ $fullName }}-pki-konnectivity-client"
         name: pki-konnectivity-client
       {{- end }}
+      {{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
+      {{ template "kubernetes.konnectivityServer.volumes" . }}
+      {{- end }}
       {{- with .Values.apiServer.extraVolumes }}
       {{- toYaml . | nindent 6 }}
       {{- end }}

diff --git a/deploy/helm/kubernetes/templates/apiserver-service.yaml b/deploy/helm/kubernetes/templates/apiserver-service.yaml
@@ -11,7 +11,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
   annotations:
-    {{- if not (index .Values.apiServer.service.annotations "metallb.universe.tf/allow-shared-ip") }}
+    {{- if and (not (index .Values.apiServer.service.annotations "metallb.universe.tf/allow-shared-ip")) (eq .Values.konnectivityServer.mode "HTTPConnect") }}
     metallb.universe.tf/allow-shared-ip: {{ $fullName }}
     {{- end }}
     {{- with .Values.apiServer.service.annotations }}
@@ -28,6 +28,13 @@ spec:
     {{- with .Values.apiServer.service.nodePort }}
     nodePort: {{ . }}
     {{- end }}
+  {{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "GRPC") }}
+  - port: {{ .Values.konnectivityServer.ports.agent }}
+    name: agent
+    {{- with .Values.konnectivityServer.service.nodePorts.client }}
+    nodePort: {{ . }}
+    {{- end }}
+  {{- end }}
   selector:
     app: {{ $fullName }}-apiserver
 {{- end }}
diff --git a/deploy/helm/kubernetes/templates/konnectivity-certs.yaml b/deploy/helm/kubernetes/templates/konnectivity-certs.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.konnectivityServer.enabled }}
+{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
 {{- $fullName := include "kubernetes.fullname" . -}}
 {{- $certName := include "kubernetes.certname" . -}}
 ---
@@ -30,6 +30,7 @@ metadata:
 spec:
   ca:
     secretName: "{{ $fullName }}-pki-konnectivity-ca"
+
 ---
 {{- $svcName1 := printf "%s-konnectivity-server" $fullName }}
 {{- $svcName2 := printf "%s-konnectivity-server.%s" $fullName .Release.Namespace }}

diff --git a/deploy/helm/kubernetes/templates/konnectivity-server-deployment.yaml b/deploy/helm/kubernetes/templates/konnectivity-server-deployment.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.konnectivityServer.enabled }}
+{{- if and .Values.konnectivityServer.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
 {{- $fullName := include "kubernetes.fullname" . -}}
 ---
 apiVersion: apps/v1
@@ -68,90 +68,9 @@ spec:
       {{- end }}
       automountServiceAccountToken: false
       containers:
-      - command:
-        - /proxy-server
-        - --logtostderr=true
-        - --server-count={{ .Values.konnectivityServer.replicaCount }}
-        - --server-id=$(POD_NAME)
-        - --cluster-cert=/pki/apiserver/tls.crt
-        - --cluster-key=/pki/apiserver/tls.key
-        - --server-ca-cert=/pki/konnectivity-server/ca.crt
-        - --server-cert=/pki/konnectivity-server/tls.crt
-        - --server-key=/pki/konnectivity-server/tls.key
-        - --mode=http-connect
-        - --server-port={{ .Values.konnectivityServer.ports.server }}
-        - --agent-port={{ .Values.konnectivityServer.ports.agent }}
-        - --admin-port={{ .Values.konnectivityServer.ports.admin }}
-        - --health-port={{ .Values.konnectivityServer.ports.health }}
-        - --agent-namespace=kube-system
-        - --agent-service-account=konnectivity-agent
-        - --kubeconfig=/etc/kubernetes/konnectivity-server.conf
-        - --authentication-audience=system:konnectivity-server
-        {{- range $key, $value := .Values.konnectivityServer.extraArgs }}
-        - --{{ $key }}={{ $value }}
-        {{- end }}
-        ports:
-        - containerPort: {{ .Values.konnectivityServer.ports.server }}
-          name: server
-        - containerPort: {{ .Values.konnectivityServer.ports.agent }}
-          name: agent
-        - containerPort: {{ .Values.konnectivityServer.ports.admin }}
-          name: admin
-        - containerPort: {{ .Values.konnectivityServer.ports.health }}
-          name: health
-        {{- with .Values.konnectivityServer.image }}
-        image: "{{ .repository }}{{ if .digest }}@{{ .digest }}{{ else }}:{{ .tag }}{{ end }}"
-        imagePullPolicy: {{ .pullPolicy }}
-        {{- end }}
-        livenessProbe:
-          failureThreshold: 8
-          httpGet:
-            path: /healthz
-            port: {{ .Values.konnectivityServer.ports.health }}
-            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 60
-        name: konnectivity-server
-        resources:
-          {{- toYaml .Values.konnectivityServer.resources | nindent 10 }}
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        {{- with .Values.konnectivityServer.extraEnv }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-        volumeMounts:
-        - mountPath: /pki/apiserver
-          name: pki-apiserver
-        - mountPath: /pki/konnectivity-server
-          name: pki-konnectivity-server
-        - mountPath: /pki/konnectivity-server-client
-          name: pki-konnectivity-server-client
-        - mountPath: /etc/kubernetes/
-          name: kubeconfig
-          readOnly: true
-        {{- with .Values.konnectivityServer.extraVolumeMounts }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
+      {{ template "kubernetes.konnectivityServer.containers" . }}
       {{- with .Values.konnectivityServer.sidecars }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
-      volumes:
-      - secret:
-          secretName: "{{ $fullName }}-pki-apiserver-server"
-        name: pki-apiserver
-      - secret:
-          secretName: "{{ $fullName }}-pki-konnectivity-server"
-        name: pki-konnectivity-server
-      - secret:
-          secretName: "{{ $fullName }}-pki-konnectivity-server-client"
-        name: pki-konnectivity-server-client
-      - configMap:
-          name: "{{ $fullName }}-konnectivity-server-conf"
-        name: kubeconfig
-      {{- with .Values.konnectivityServer.extraVolumes }}
-      {{- toYaml . | nindent 6 }}
-      {{- end }}
+      {{ template "kubernetes.konnectivityServer.volumes" . }}
 {{- end }}
diff --git a/deploy/helm/kubernetes/templates/konnectivity-server-service.yaml b/deploy/helm/kubernetes/templates/konnectivity-server-service.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled }}
+{{- if and .Values.konnectivityServer.enabled .Values.konnectivityServer.service.enabled (eq .Values.konnectivityServer.mode "HTTPConnect") }}
 {{- $fullName := include "kubernetes.fullname" . -}}
 ---
 apiVersion: v1

diff --git a/deploy/helm/kubernetes/values.yaml b/deploy/helm/kubernetes/values.yaml
@@ -287,6 +287,11 @@ coredns:
 
 konnectivityServer:
   enabled: false
+  # This controls the protocol between the API Server and the Konnectivity server.
+  # Supported values are "GRPC" and "HTTPConnect".
+  # "GRPC" will deploy konnectivity-server as a sidecar for apiserver
+  # "HTTPConnect" will deploy konnectivity-server as separate deployment
+  mode: GRPC
   image:
     repository: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server
     tag: v0.0.20