Skip to content

Commit

Permalink
feat: support internal load balancer type (#147)
Browse files Browse the repository at this point in the history 
feat: support internal lb
  • Loading branch information
@zhiying-lin
zhiying-lin committed Dec 7, 2022
1 parent 318a74e commit 5e3ea49
Show file tree
Hide file tree
Showing 3 changed files with 242 additions and 0 deletions.
19 changes: 19 additions & 0 deletions pkg/controllers/multiclusterservice/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package multiclusterservice
import (
"context"
"fmt"
"strconv"
"time"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -49,6 +50,12 @@ const (

// ControllerName is the name of the Reconciler.
ControllerName = "multiclusterservice-controller"

// multiClusterService annotation
multiClusterServiceAnnotationInternalLoadBalancer = "networking.fleet.azure.com/azure-load-balancer-internal"

// service annotation
serviceAnnotationInternalLoadBalancer = "service.beta.kubernetes.io/azure-load-balancer-internal"
)

// Reconciler reconciles a MultiClusterService object.
Expand Down Expand Up @@ -335,6 +342,17 @@ func (r *Reconciler) updateMultiClusterLabel(ctx context.Context, mcs *fleetnetv
return nil
}

func configureInternalLoadBalancer(mcs *fleetnetv1alpha1.MultiClusterService, service *corev1.Service) {
isInternal, err := strconv.ParseBool(mcs.Annotations[multiClusterServiceAnnotationInternalLoadBalancer])
if err != nil || !isInternal {
return
}
if service.GetAnnotations() == nil { // in case annotation map is nil
service.Annotations = map[string]string{}
}
service.Annotations[serviceAnnotationInternalLoadBalancer] = "true"
}

func (r *Reconciler) ensureDerivedService(mcs *fleetnetv1alpha1.MultiClusterService, serviceImport *fleetnetv1alpha1.ServiceImport, service *corev1.Service) error {
svcPorts := make([]corev1.ServicePort, len(serviceImport.Status.Ports))
for i, importPort := range serviceImport.Status.Ports {
Expand All @@ -349,6 +367,7 @@ func (r *Reconciler) ensureDerivedService(mcs *fleetnetv1alpha1.MultiClusterServ

service.Labels[serviceLabelMCSName] = mcs.Name
service.Labels[serviceLabelMCSNamespace] = mcs.Namespace
configureInternalLoadBalancer(mcs, service)
return nil
}

Expand Down
136 changes: 136 additions & 0 deletions pkg/controllers/multiclusterservice/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ func TestHandleUpdate(t *testing.T) {
tests := []struct {
name string
labels map[string]string
annotations map[string]string
status *fleetnetv1alpha1.MultiClusterServiceStatus
serviceImport *fleetnetv1alpha1.ServiceImport
hasOldServiceImport bool
Expand Down Expand Up @@ -918,13 +919,91 @@ func TestHandleUpdate(t *testing.T) {
},
},
},
{
name: "no updates on the mcs with internal load balancer type (valid service import) without derived service resource",
labels: map[string]string{
multiClusterServiceLabelServiceImport: testServiceName,
objectmeta.MultiClusterServiceLabelDerivedService: derivedServiceName,
},
annotations: map[string]string{
multiClusterServiceAnnotationInternalLoadBalancer: "true",
},
serviceImport: &fleetnetv1alpha1.ServiceImport{
ObjectMeta: metav1.ObjectMeta{
Name: testServiceName,
Namespace: testNamespace,
},
Status: fleetnetv1alpha1.ServiceImportStatus{
Ports: importServicePorts,
Clusters: []fleetnetv1alpha1.ClusterStatus{
{Cluster: "member1"},
},
},
},
want: ctrl.Result{},
wantServiceImport: &fleetnetv1alpha1.ServiceImport{
TypeMeta: serviceImportType,
ObjectMeta: metav1.ObjectMeta{
Name: testServiceName,
Namespace: testNamespace,
OwnerReferences: []metav1.OwnerReference{ownerRef},
},
Status: fleetnetv1alpha1.ServiceImportStatus{
Ports: importServicePorts,
Clusters: []fleetnetv1alpha1.ClusterStatus{
{Cluster: "member1"},
},
},
},
wantDerivedService: &corev1.Service{
TypeMeta: serviceType,
ObjectMeta: metav1.ObjectMeta{
Name: derivedServiceName,
Namespace: systemNamespace,
Labels: serviceLabel,
Annotations: map[string]string{
serviceAnnotationInternalLoadBalancer: "true",
},
},
Spec: corev1.ServiceSpec{
Ports: servicePorts,
Type: corev1.ServiceTypeLoadBalancer,
},
},
wantMCS: &fleetnetv1alpha1.MultiClusterService{
TypeMeta: multiClusterServiceType,
ObjectMeta: metav1.ObjectMeta{
Name: testName,
Namespace: testNamespace,
Labels: map[string]string{
multiClusterServiceLabelServiceImport: testServiceName,
objectmeta.MultiClusterServiceLabelDerivedService: derivedServiceName,
},
Annotations: map[string]string{
multiClusterServiceAnnotationInternalLoadBalancer: "true",
},
},
Spec: fleetnetv1alpha1.MultiClusterServiceSpec{
ServiceImport: fleetnetv1alpha1.ServiceImportRef{
Name: testServiceName,
},
},
Status: fleetnetv1alpha1.MultiClusterServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{},
Conditions: []metav1.Condition{
validCondition,
},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()

mcsObj := multiClusterServiceForTest()
mcsObj.ObjectMeta.Labels = tc.labels
mcsObj.ObjectMeta.Annotations = tc.annotations
if tc.status != nil {
mcsObj.Status = *tc.status
}
Expand Down Expand Up @@ -993,3 +1072,60 @@ func TestHandleUpdate(t *testing.T) {
})
}
}

func TestConfigureInternalLoadBalancer(t *testing.T) {
tests := []struct {
name string
annotations map[string]string
want map[string]string
}{
{
name: "annotations are nil",
},
{
name: "internal load balancer annotation is not set",
annotations: map[string]string{
"azure-load-balancer-internal": "true",
},
},
{
name: "internal load balancer annotation is set",
annotations: map[string]string{
multiClusterServiceAnnotationInternalLoadBalancer: "true",
},
want: map[string]string{
serviceAnnotationInternalLoadBalancer: "true",
},
},
{
name: "internal load balancer annotation is set as false",
annotations: map[string]string{
multiClusterServiceAnnotationInternalLoadBalancer: "false",
},
},
{
name: "internal load balancer annotation is set as invalid value",
annotations: map[string]string{
multiClusterServiceAnnotationInternalLoadBalancer: "falsse",
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
mcs := &fleetnetv1alpha1.MultiClusterService{
ObjectMeta: metav1.ObjectMeta{
Annotations: tc.annotations,
},
}
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: tc.want,
},
}
configureInternalLoadBalancer(mcs, service)
if got := service.GetAnnotations(); !cmp.Equal(got, tc.want) {
t.Errorf("configureInternalLoadBalancer() got service annotations %+v, want %+v", got, tc.want)
}
})
}
}
87 changes: 87 additions & 0 deletions test/e2e/export_service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,93 @@ var _ = Describe("Test exporting service", func() {
Expect(memberClusterMCS.Client().Delete(ctx, &newMCSDef)).Should(Succeed(), "Failed to delete multi-cluster service %s in cluster %s", multiClusterSvcKey, memberClusterMCS.Name())
})
})

Context("Test multi-cluster service with internal load balancer type", func() {
BeforeEach(func() {
By("Exporting the service")
Expect(wm.ExportService(ctx, wm.ServiceExport())).Should(Succeed())
})
AfterEach(func() {
By("Unexporting the service")
Expect(wm.UnexportService(ctx, wm.ServiceExport())).Should(Succeed())
})

It("should allow multi-cluster service with internal load balancer type", func() {
By("Updating multi-cluster service to import the new service")
mcs := wm.MultiClusterService()
mcs.Annotations = map[string]string{
"networking.fleet.azure.com/azure-load-balancer-internal": "true",
}
By("Creating multi-cluster service")
Expect(wm.CreateMultiClusterService(ctx, mcs)).Should(Succeed())

By("Validating the multi-cluster service")
memberClusterMCS := wm.Fleet.MCSMemberCluster()

By("Validating the multi-cluster service is importing a service")
multiClusterSvcKey := types.NamespacedName{Namespace: mcs.Namespace, Name: mcs.Name}
mcsObj := &fleetnetv1alpha1.MultiClusterService{}
Eventually(func() string {
if err := memberClusterMCS.Client().Get(ctx, multiClusterSvcKey, mcsObj); err != nil {
return err.Error()
}
wantedMCSCondition := []metav1.Condition{
{
Type: string(fleetnetv1alpha1.MultiClusterServiceValid),
Reason: "FoundServiceImport",
Status: metav1.ConditionTrue,
},
}
return cmp.Diff(wantedMCSCondition, mcsObj.Status.Conditions, framework.MCSConditionCmpOptions...)
}, framework.PollTimeout, framework.PollInterval).Should(BeEmpty(), "Validate multi-cluster service condition mismatch (-want, +got):")

By("Validating the multi-cluster service is taking the service spec")
derivedServiceName := mcsObj.GetLabels()["networking.fleet.azure.com/derived-service"]
derivedServiceObj := &corev1.Service{}
svc := wm.Service()
Eventually(func() string {
derivedServiceKey := types.NamespacedName{Namespace: fleetSystemNamespace, Name: derivedServiceName}
if err := memberClusterMCS.Client().Get(ctx, derivedServiceKey, derivedServiceObj); err != nil {
return err.Error()
}
wantedDerivedSvcPortSpec := []corev1.ServicePort{
{
Port: svc.Spec.Ports[0].Port,
TargetPort: svc.Spec.Ports[0].TargetPort,
},
}
derivedSvcPortSpecCmpOptions := []cmp.Option{
cmpopts.IgnoreFields(corev1.ServicePort{}, "NodePort", "Protocol"),
}
return cmp.Diff(wantedDerivedSvcPortSpec, derivedServiceObj.Spec.Ports, derivedSvcPortSpecCmpOptions...)

}, framework.PollTimeout, framework.PollInterval).Should(BeEmpty(), "Validate derived service port spec (-want, +got):")
wantedDerivedSvcAnnotation := map[string]string{
"service.beta.kubernetes.io/azure-load-balancer-internal": "true",
}
Expect(cmp.Equal(derivedServiceObj.Annotations, wantedDerivedSvcAnnotation)).Should(BeTrue(),
"Validate derived service annotation, got %+v, want %+v", derivedServiceObj.Annotations, wantedDerivedSvcAnnotation)

By("Validating the multi-cluster service has loadbalancer ingress IP address")
Eventually(func() error {
if err := memberClusterMCS.Client().Get(ctx, multiClusterSvcKey, mcsObj); err != nil {
return err
}
if len(mcsObj.Status.LoadBalancer.Ingress) != 1 {
return fmt.Errorf("multi-cluster service ingress address length, got %d, want %d", 0, 1)
}
if mcsObj.Status.LoadBalancer.Ingress[0].IP == "" {
return fmt.Errorf("multi-cluster service load balancer IP, got empty, want not empty")
}
return nil
}, framework.MCSLBPollTimeout, framework.PollInterval).Should(Succeed(), "Failed to retrieve multi-cluster service LB address")

// clean up
By("Deleting multi-cluster service")
Expect(wm.DeleteMultiClusterService(ctx, mcs)).Should(Succeed())
})

})
})

func fetchHTTPRequestBody(requestURL string) (string, error) {
Expand Down

0 comments on commit 5e3ea49

Please sign in to comment.