test: add sustained workload scenario to the performance test suite

test/e2e/framework/cluster.go

@@ -12,10 +12,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -28,7 +28,7 @@ type Cluster struct {
 	prometheusAPIServiceAddr string
 }
 
-// NewCluster creates Cluster and initializes its kubernetes client.
+// NewCluster creates a Cluster and initializes its Kubernetes client.
 func NewCluster(name string, scheme *runtime.Scheme) (*Cluster, error) {
 	cluster := &Cluster{
 		scheme: scheme,
@@ -40,6 +40,18 @@ func NewCluster(name string, scheme *runtime.Scheme) (*Cluster, error) {
 	return cluster, nil
 }
 
+// NewClusterWithCustomQPS creates a Cluster and initializes its Kubernetes client with custom QPS and Burst settings.
+func NewClusterWithCustomQPS(name string, scheme *runtime.Scheme, QPS, BurstQPS int) (*Cluster, error) {
+	cluster := &Cluster{
+		scheme: scheme,
+		name:   name,
+	}
+	if err := cluster.initClusterClientWithCustomQPS(QPS, BurstQPS); err != nil {
+		return nil, err
+	}
+	return cluster, nil
+}
+
 // Name returns the cluster name.
 func (c *Cluster) Name() string {
 	return c.name
@@ -78,16 +90,25 @@ func (c *Cluster) SetupPrometheusAPIServiceAccess(ctx context.Context, prometheu
 }
 
 func (c *Cluster) initClusterClient() error {
-	clusterConfig, err := c.buildClientConfig()
+	restConfig, err := c.retrieveRESTConfig()
 	if err != nil {
 		return err
 	}
-
-	restConfig, err := clusterConfig.ClientConfig()
+	kubeClient, err := client.New(restConfig, client.Options{Scheme: c.scheme})
 	if err != nil {
-		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		return err
 	}
+	c.kubeClient = kubeClient
+	return nil
+}
 
+func (c *Cluster) initClusterClientWithCustomQPS(QPS, BurstQPS int) error {
+	restConfig, err := c.retrieveRESTConfig()
+	if err != nil {
+		return err
+	}
+	restConfig.QPS = float32(QPS)
+	restConfig.Burst = BurstQPS
 	kubeClient, err := client.New(restConfig, client.Options{Scheme: c.scheme})
 	if err != nil {
 		return err
@@ -96,6 +117,19 @@ func (c *Cluster) initClusterClient() error {
 	return nil
 }
 
+func (c *Cluster) retrieveRESTConfig() (*rest.Config, error) {
+	clusterConfig, err := c.buildClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	restConfig, err := clusterConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	return restConfig, nil
+}
+
 func (c *Cluster) buildClientConfig() (clientcmd.ClientConfig, error) {
 	kubeConfig, err := fetchKubeConfig()
 	if err != nil {

test/perftest/README.md

@@ -1,4 +1,4 @@
-# Fleet Networking Performance Test
+# Fleet Networking Performance Test Suite
 
 This package features the performance test suite for Fleet networking controllers.
 
@@ -16,16 +16,24 @@ To run this test:
 
     It should take approximately 40 minutes to complete the setup.
 
-2. Run the test suite with Ginkgo:
+2. Pick a test suite and run it with Ginkgo:
 
     ```sh
-    go test ./test/perftest/ --ginkgo.v -test.v -timeout 1h | tee perf_test.log
+    # Replace TEST_SUITE_PATH with a value of your own
+    go test TEST_SUITE_PATH --ginkgo.v -test.v -timeout 1h | tee perf_test.log
     ```
 
-    It should take approximately 20 minutes to finish running the test suite.
+    The list of currently available test suites are:
 
-After finishing the suite, you should be able to find the service export latency and the `endpointSlice`
-export/import latency in the output. To collect other metrics for further analysis:
+    | Path        | Description |
+    | ----------- | ----------- |
+    | `test/perftest/latency/peak` | A test suite for evaluating export/import latencies under peak workloads |
+    | `test/perftest/latency/sustained` | A test suite for evaluating export/import latencies under sustained workloads |
+
+    It is strongly recommended that **each test suite runs in its own environment**.
+
+After finishing the suite, find the results, e.g., service export latencies and `endpointSlice`
+export/import latencies in the output and/or the log file. To collect other metrics for further analysis:
 
 1. Pick a cluster and find the public IP address for the Prometheus dashboard:
 
@@ -39,8 +47,7 @@ export/import latency in the output. To collect other metrics for further analys
     In the list of services, write down the external IP address of the service `metrics-dashboard`.
 
 2. Open a browser and type in the address, `http://YOUR-IP-ADDRESS:9090` (replace `YOUR-IP-ADDRESS` with the public
-IP of the `metrics-dashboard` service). You should be able to run any query in the dashboard to retrieve data for
-the specific cluster.
+IP of the `metrics-dashboard` service). The Prometheus dashboard will open for metric queries.
 
     A few metrics that might be of particular interests in this test suite are:
 
@@ -49,5 +56,4 @@ the specific cluster.
     * `workqueue_queue_duration_seconds` (a Prometheus histogram)
 
     Note that due to AKS's architecture, metrics about the Kubernetes API server, the etcd backend, and a number of
-    other control plane components are not available through the `kube-prometheus-stack` we install; you should
-    be able to find these data through AKS instead.
+    other control plane components are not available through the `kube-prometheus-stack` installed; find these data through AKS instead.

test/perftest/suite_test.go → test/perftest/latency/peak/suite_test.go

@@ -3,7 +3,8 @@ Copyright (c) Microsoft Corporation.
 Licensed under the MIT license.
 */
 
-// Package perftest features the performance test suite for Fleet networking controllers.
+// Package sustained features the performance test suite for evaluating Fleet networking related latencies under
+// peak loads.
 package perftest
 
 import (