feat: Add validation webhook scaffolding code (#39)

Co-authored-by: guofei <[email protected]>

api/v1alpha1/workspace_default.go

@@ -0,0 +1,9 @@
+package v1alpha1
+
+import (
+	"context"
+)
+
+// SetDefaults for the Workspace
+func (w *Workspace) SetDefaults(_ context.Context) {
+}

api/v1alpha1/workspace_validation.go

@@ -0,0 +1,23 @@
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	"knative.dev/pkg/apis"
+)
+
+func (w *Workspace) SupportedVerbs() []admissionregistrationv1.OperationType {
+	return []admissionregistrationv1.OperationType{
+		admissionregistrationv1.Create,
+		admissionregistrationv1.Update,
+	}
+}
+
+func (w *Workspace) Validate(ctx context.Context) (errs *apis.FieldError) {
+	klog.InfoS("Validating", "workspace", fmt.Sprintf("%s/%s", w.Namespace, w.Name))
+	return errs
+}

charts/kdm/templates/clusterrole.yaml

@@ -30,3 +30,10 @@ rules:
   - apiGroups: ["karpenter.sh"]
     resources: ["machines", "machines/status"]
     verbs: ["get","list","watch","create", "delete", "update", "patch"]
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources: ["validatingwebhookconfigurations"]
+    verbs: ["get","list","watch"]
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources: ["validatingwebhookconfigurations"]
+    verbs: ["update"]
+    resourceNames: ["validation.webhook.kdm.io"]

charts/kdm/templates/deployment.yaml

@@ -39,10 +39,25 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: WEBHOOK_SERVICE
+              value: {{ include "kdm.fullname" . }}
+            - name: WEBHOOK_PORT
+              value: "{{ .Values.webhook.port }}"
+            - name: SYSTEM_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
           ports:
             - name: http
               containerPort: 80
               protocol: TCP
+            - name: http-metrics
+              containerPort: 8080
+              protocol: TCP
+            - name: https-webhook
+              containerPort: {{ .Values.webhook.port }}
+              protocol: TCP
           livenessProbe:
             httpGet:
               path: /healthz

charts/kdm/templates/role.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "kdm.fullname" . }}-role
+  namespace: {{ include "kdm.fullname" .}}
+  labels:
+    {{- include "kdm.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get","list","watch","create", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get","list","watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["update"]
+    resourceNames: ["webhook-cert"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["patch", "update"]
+    resourceNames: ["webhook.validationwebhook.00-of-01", "webhook.webhookcertificates.00-of-01"]

charts/kdm/templates/role_binding.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "kdm.fullname" . }}-rolebinding
+  namespace: {{ include "kdm.fullname" . }}
+  labels:
+   {{- include "kdm.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kdm.fullname" . }}-role
+subjects:
+- kind: ServiceAccount
+  name: {{ include "kdm.fullname" . }}-sa
+  namespace: {{ include "kdm.fullname" . }}

charts/kdm/templates/secret-webhook-cert.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: webhook-cert
+  namespace: {{ include "kdm.fullname" .}}
+  labels:
+    {{- include "kdm.labels" . | nindent 4 }}
+data:
+   server-key.pem: ""
+   server-cert.pem: ""
+   ca-cert.pem: ""

charts/kdm/templates/service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "kdm.fullname" . }}
+  namespace: {{ include "kdm.fullname" .}}
+  labels:
+    {{- include "kdm.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http-metrics
+      port: 8080 
+      targetPort: http-metrics
+      protocol: TCP
+    - name: https-webhook
+      port: {{ .Values.webhook.port }}
+      targetPort: https-webhook
+      protocol: TCP
+  selector:
+    {{- include "kdm.selectorLabels" . | nindent 4 }}

charts/kdm/templates/webhooks.yaml

@@ -0,0 +1,26 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validation.webhook.kdm.io
+  labels:
+    {{- include "kdm.labels" . | nindent 4 }}
+webhooks:
+  - name: validation.webhook.kdm.io
+    admissionReviewVersions: ["v1"]
+    clientConfig:
+      service:
+        name: {{ include "kdm.fullname" . }}
+        namespace: {{ include "kdm.fullname" . }}
+        port: {{ .Values.webhook.port }}
+    failurePolicy: Fail
+    sideEffects: None
+    rules:
+      - apiGroups:
+          - kdm.io
+        apiVersions:
+          - v1alpha1
+        resources:
+          - workspaces
+        operations:
+          - CREATE
+          - UPDATE

charts/kdm/values.yaml

@@ -22,6 +22,9 @@ securityContext:
     drop:
       - "ALL"
 
+webhook:
+  port: 9443
+
 resources:
   limits:
     cpu: 500m

cmd/main.go

@@ -19,11 +19,16 @@ package main
 import (
 	"flag"
 	"os"
+	"strconv"
+	"time"
 
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
 	"github.com/kdm/pkg/controllers"
+	"github.com/kdm/pkg/webhooks"
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"knative.dev/pkg/injection/sharedmain"
+	"knative.dev/pkg/signals"
+	"knative.dev/pkg/webhook"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -41,6 +46,11 @@ import (
 	//+kubebuilder:scaffold:imports
 )
 
+const (
+	WebhookServiceName = "WEBHOOK_SERVICE"
+	WebhookServicePort = "WEBHOOK_PORT"
+)
+
 var (
 	scheme = runtime.NewScheme()
 
@@ -62,12 +72,15 @@ func init() {
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
+	var enableWebhook bool
 	var probeAddr string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.BoolVar(&enableWebhook, "webhook", true,
+		"Enable webhook for controller manager. Default is true.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -77,11 +90,8 @@ func main() {
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:             scheme,
-		MetricsBindAddress: metricsAddr,
-		WebhookServer: webhook.NewServer(webhook.Options{
-			Port: 9443,
-		}),
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "ef60f9b0.io",
@@ -103,8 +113,10 @@ func main() {
 	}
 
 	if err = (&controllers.WorkspaceReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:   mgr.GetClient(),
+		Log:      log.Log.WithName("controllers").WithName("Workspace"),
+		Scheme:   mgr.GetScheme(),
+		Recorder: mgr.GetEventRecorderFor("KDM-Workspace-controller"),
 	}).SetupWithManager(mgr); err != nil {
 		klog.ErrorS(err, "unable to create controller", "controller", "Workspace")
 		exitWithErrorFunc()
@@ -120,19 +132,28 @@ func main() {
 		exitWithErrorFunc()
 	}
 
+	if enableWebhook {
+		klog.InfoS("starting webhook reconcilers")
+		go func() {
+			p, err := strconv.Atoi(os.Getenv(WebhookServicePort))
+			if err != nil {
+				klog.ErrorS(err, "unable to parse the webhook port number")
+				exitWithErrorFunc()
+			}
+			ctx := webhook.WithOptions(signals.NewContext(), webhook.Options{
+				ServiceName: os.Getenv(WebhookServiceName),
+				Port:        p,
+				SecretName:  "webhook-cert",
+			})
+			sharedmain.MainWithConfig(sharedmain.WithHealthProbesDisabled(ctx), "webhook", ctrl.GetConfigOrDie(), webhooks.NewWebhooks()...)
+		}()
+		// wait 2 seconds to allow reconciling webhookconfiguration and service endpoint.
+		time.Sleep(2 * time.Second)
+	}
+
 	klog.InfoS("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		klog.ErrorS(err, "problem running manager")
 		exitWithErrorFunc()
 	}
-
-	workspaceController := &controllers.WorkspaceReconciler{
-		Client:   mgr.GetClient(),
-		Log:      log.Log.WithName("controllers").WithName("Workspace"),
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor("KDM-Workspace-controller"),
-	}
-	if err := workspaceController.SetupWithManager(mgr); err != nil {
-		// TODO Handle error
-	}
 }

go.mod

@@ -13,32 +13,43 @@ require (
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kubernetes v1.27.2
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491
-	knative.dev/pkg v0.0.0-20230502134655-db8a35330281
+	knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4
 	sigs.k8s.io/controller-runtime v0.15.2
 )
 
 require (
+	contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect
+	contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/blendle/zapdriver v1.3.1 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.1 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gobuffalo/flect v0.2.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -50,20 +61,27 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/statsd_exporter v0.21.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/automaxprocs v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
 	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
+	google.golang.org/api v0.124.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect