Skip to content

Commit

Permalink
Added a performance test suite
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelawyu committed Nov 25, 2022
1 parent f126547 commit a787fdd
Show file tree
Hide file tree
Showing 7 changed files with 505 additions and 18 deletions.
46 changes: 40 additions & 6 deletions test/e2e/framework/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ import (
"os"
"path/filepath"

"github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand All @@ -28,7 +28,7 @@ type Cluster struct {
prometheusAPIServiceAddr string
}

// NewCluster creates Cluster and initializes its kubernetes client.
// NewCluster creates a Cluster and initializes its Kubernetes client.
func NewCluster(name string, scheme *runtime.Scheme) (*Cluster, error) {
cluster := &Cluster{
scheme: scheme,
Expand All @@ -40,6 +40,18 @@ func NewCluster(name string, scheme *runtime.Scheme) (*Cluster, error) {
return cluster, nil
}

// NewClusterWithCustomQPS creates a Cluster and initializes its Kubernetes client with custom QPS and Burst settings.
func NewClusterWithCustomQPS(name string, scheme *runtime.Scheme, QPS, BurstQPS int) (*Cluster, error) {
cluster := &Cluster{
scheme: scheme,
name: name,
}
if err := cluster.initClusterClientWithCustomQPS(QPS, BurstQPS); err != nil {
return nil, err
}
return cluster, nil
}

// Name returns the cluster name.
func (c *Cluster) Name() string {
return c.name
Expand Down Expand Up @@ -78,16 +90,25 @@ func (c *Cluster) SetupPrometheusAPIServiceAccess(ctx context.Context, prometheu
}

func (c *Cluster) initClusterClient() error {
clusterConfig, err := c.buildClientConfig()
restConfig, err := c.retrieveRESTConfig()
if err != nil {
return err
}

restConfig, err := clusterConfig.ClientConfig()
kubeClient, err := client.New(restConfig, client.Options{Scheme: c.scheme})
if err != nil {
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
return err
}
c.kubeClient = kubeClient
return nil
}

func (c *Cluster) initClusterClientWithCustomQPS(QPS, BurstQPS int) error {
restConfig, err := c.retrieveRESTConfig()
if err != nil {
return err
}
restConfig.QPS = float32(QPS)
restConfig.Burst = BurstQPS
kubeClient, err := client.New(restConfig, client.Options{Scheme: c.scheme})
if err != nil {
return err
Expand All @@ -96,6 +117,19 @@ func (c *Cluster) initClusterClient() error {
return nil
}

func (c *Cluster) retrieveRESTConfig() (*rest.Config, error) {
clusterConfig, err := c.buildClientConfig()
if err != nil {
return nil, err
}

restConfig, err := clusterConfig.ClientConfig()
if err != nil {
return nil, err
}
return restConfig, nil
}

func (c *Cluster) buildClientConfig() (clientcmd.ClientConfig, error) {
kubeConfig, err := fetchKubeConfig()
if err != nil {
Expand Down
26 changes: 16 additions & 10 deletions test/perftest/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Fleet Networking Performance Test
# Fleet Networking Performance Test Suite

This package features the performance test suite for Fleet networking controllers.

Expand All @@ -16,16 +16,24 @@ To run this test:

It should take approximately 40 minutes to complete the setup.

2. Run the test suite with Ginkgo:
2. Pick a test suite and run it with Ginkgo:

```sh
go test ./test/perftest/ --ginkgo.v -test.v -timeout 1h | tee perf_test.log
# Replace TEST_SUITE_PATH with a value of your own
go test TEST_SUITE_PATH --ginkgo.v -test.v -timeout 1h | tee perf_test.log
```

It should take approximately 20 minutes to finish running the test suite.
The list of currently available test suites are:

After finishing the suite, you should be able to find the service export latency and the `endpointSlice`
export/import latency in the output. To collect other metrics for further analysis:
| Path | Description |
| ----------- | ----------- |
| `test/perftest/latency/peak` | A test suite for evaluating export/import latencies under peak workloads |
| `test/perftest/latency/sustained` | A test suite for evaluating export/import latencies under sustained workloads |

It is strongly recommended that **each test suite runs in its own environment**.

After finishing the suite, find the results, e.g., service export latencies and `endpointSlice`
export/import latencies in the output and/or the log file. To collect other metrics for further analysis:

1. Pick a cluster and find the public IP address for the Prometheus dashboard:

Expand All @@ -39,8 +47,7 @@ export/import latency in the output. To collect other metrics for further analys
In the list of services, write down the external IP address of the service `metrics-dashboard`.

2. Open a browser and type in the address, `http://YOUR-IP-ADDRESS:9090` (replace `YOUR-IP-ADDRESS` with the public
IP of the `metrics-dashboard` service). You should be able to run any query in the dashboard to retrieve data for
the specific cluster.
IP of the `metrics-dashboard` service). The Prometheus dashboard will open for metric queries.

A few metrics that might be of particular interests in this test suite are:

Expand All @@ -49,5 +56,4 @@ the specific cluster.
* `workqueue_queue_duration_seconds` (a Prometheus histogram)

Note that due to AKS's architecture, metrics about the Kubernetes API server, the etcd backend, and a number of
other control plane components are not available through the `kube-prometheus-stack` we install; you should
be able to find these data through AKS instead.
other control plane components are not available through the `kube-prometheus-stack` installed; find these data through AKS instead.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ Copyright (c) Microsoft Corporation.
Licensed under the MIT license.
*/

// Package perftest features the performance test suite for Fleet networking controllers.
// Package sustained features the performance test suite for evaluating Fleet networking related latencies under
// peak loads.
package perftest

import (
Expand Down
Loading

0 comments on commit a787fdd

Please sign in to comment.