From a2014943b5c960f6d125bc4267536926665a25a4 Mon Sep 17 00:00:00 2001 From: Candace Holman Date: Mon, 23 Jan 2023 11:32:11 -0500 Subject: [PATCH] Issue #1579 TLSRoute Passthrough Conformance Test (normative) (#1587) * Issue #1579 TLSRoute Passthrough Conformance Test (normative) rebase * Issue #1579 TLSRoute Passthrough - PR review update - rebase * Issue #1579 TLSRoute Passthrough - PR review update - latest --- conformance/base/manifests.yaml | 62 +++++++++++++ conformance/conformance_test.go | 5 +- .../tests/tlsroute-simple-same-namespace.go | 88 ++++++++++++++++++ .../tests/tlsroute-simple-same-namespace.yaml | 35 +++++++ conformance/utils/config/timeout.go | 16 +++- conformance/utils/http/http.go | 32 ++++--- conformance/utils/kubernetes/certificate.go | 2 +- conformance/utils/kubernetes/helpers.go | 93 ++++++++++++++++++- .../utils/roundtripper/roundtripper.go | 44 ++++++++- conformance/utils/suite/suite.go | 2 + conformance/utils/tls/tls.go | 64 +++++++++++++ site-src/concepts/conformance.md | 22 ++++- 12 files changed, 429 insertions(+), 36 deletions(-) create mode 100644 conformance/tests/tlsroute-simple-same-namespace.go create mode 100644 conformance/tests/tlsroute-simple-same-namespace.yaml create mode 100644 conformance/utils/tls/tls.go diff --git a/conformance/base/manifests.yaml b/conformance/base/manifests.yaml index ae2a5e3f06..7ab20d575f 100644 --- a/conformance/base/manifests.yaml +++ b/conformance/base/manifests.yaml @@ -201,6 +201,68 @@ spec: cpu: 10m --- apiVersion: v1 +kind: Service +metadata: + name: tls-backend + namespace: gateway-conformance-infra +spec: + selector: + app: tls-backend + ports: + - protocol: TCP + port: 443 + targetPort: 8443 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tls-backend + namespace: gateway-conformance-infra + labels: + app: tls-backend +spec: + replicas: 1 + selector: + matchLabels: + app: tls-backend + template: + metadata: + labels: + app: tls-backend + spec: + containers: + - name: tls-backend + image: gcr.io/k8s-staging-ingressconformance/echoserver:v20221109-7ee2f3e + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TLS_SERVER_CERT + value: /etc/secret-volume/crt + - name: TLS_SERVER_PRIVKEY + value: /etc/secret-volume/key + resources: + requests: + cpu: 10m + volumes: + - name: secret-volume + secret: + secretName: tls-passthrough-checks-certificate + items: + - key: tls.crt + path: crt + - key: tls.key + path: key +--- +apiVersion: v1 kind: Namespace metadata: name: gateway-conformance-app-backend diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go index e9fc78ea27..91a788c667 100644 --- a/conformance/conformance_test.go +++ b/conformance/conformance_test.go @@ -44,14 +44,15 @@ func TestConformance(t *testing.T) { v1alpha2.AddToScheme(client.Scheme()) v1beta1.AddToScheme(client.Scheme()) - t.Logf("Running conformance tests with %s GatewayClass", *flags.GatewayClassName) - supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures) exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures) for feature := range exemptFeatures { supportedFeatures[feature] = false } + t.Logf("Running conformance tests with %s GatewayClass\n cleanup: %t\n debug: %t\n supported features: [%v]\n exempt features: [%v]", + *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.SupportedFeatures, *flags.ExemptFeatures) + cSuite := suite.New(suite.Options{ Client: client, GatewayClassName: *flags.GatewayClassName, diff --git a/conformance/tests/tlsroute-simple-same-namespace.go b/conformance/tests/tlsroute-simple-same-namespace.go new file mode 100644 index 0000000000..9002c16e5f --- /dev/null +++ b/conformance/tests/tlsroute-simple-same-namespace.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "context" + "fmt" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/gateway-api/apis/v1beta1" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/conformance/utils/tls" +) + +func init() { + ConformanceTests = append(ConformanceTests, TLSRouteSimpleSameNamespace) +} + +var TLSRouteSimpleSameNamespace = suite.ConformanceTest{ + ShortName: "TLSRouteSimpleSameNamespace", + Description: "A single TLSRoute in the gateway-conformance-infra namespace attaches to a Gateway in the same namespace", + Manifests: []string{"tests/tlsroute-simple-same-namespace.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := v1beta1.Namespace("gateway-conformance-infra") + routeNN := types.NamespacedName{Name: "gateway-conformance-infra-test", Namespace: string(ns)} + gwNN := types.NamespacedName{Name: "gateway-tlsroute", Namespace: string(ns)} + certNN := types.NamespacedName{Name: "tls-passthrough-checks-certificate", Namespace: string(ns)} + + gwAddr, hostnames := kubernetes.GatewayAndTLSRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + if len(hostnames) != 1 { + t.Fatalf("unexpected error in test configuration, found %d hostnames", len(hostnames)) + } + serverStr := string(hostnames[0]) + + cPem, keyPem, err := GetTLSSecret(suite.Client, certNN) + if err != nil { + t.Fatalf("unexpected error finding TLS secret: %v", err) + } + t.Run("Simple TLS request matching TLSRoute should reach infra-backend", func(t *testing.T) { + tls.MakeTLSRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, cPem, keyPem, serverStr, + http.ExpectedResponse{ + Request: http.Request{Host: serverStr, Path: "/"}, + Backend: "tls-backend", + Namespace: "gateway-conformance-infra", + }) + }) + }, +} + +// GetTLSSecret fetches the named Secret and converts both cert and key to []byte +func GetTLSSecret(client client.Client, secretName types.NamespacedName) ([]byte, []byte, error) { + var cert, key []byte + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + secret := &v1.Secret{} + err := client.Get(ctx, secretName, secret) + if err != nil { + return cert, key, fmt.Errorf("error fetching TLS Secret: %w", err) + } + cert = secret.Data["tls.crt"] + key = secret.Data["tls.key"] + + return cert, key, nil +} diff --git a/conformance/tests/tlsroute-simple-same-namespace.yaml b/conformance/tests/tlsroute-simple-same-namespace.yaml new file mode 100644 index 0000000000..fa50abe571 --- /dev/null +++ b/conformance/tests/tlsroute-simple-same-namespace.yaml @@ -0,0 +1,35 @@ +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TLSRoute +metadata: + name: gateway-conformance-infra-test + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: gateway-tlsroute + namespace: gateway-conformance-infra + hostnames: + - abc.example.com + rules: + - backendRefs: + - name: tls-backend + port: 443 +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: gateway-tlsroute + namespace: gateway-conformance-infra +spec: + gatewayClassName: "{GATEWAY_CLASS_NAME}" + listeners: + - name: https + port: 443 + protocol: TLS + hostname: "*.example.com" + allowedRoutes: + namespaces: + from: Same + kinds: + - kind: TLSRoute + tls: + mode: Passthrough diff --git a/conformance/utils/config/timeout.go b/conformance/utils/config/timeout.go index 039e4e4e6a..d1064e7066 100644 --- a/conformance/utils/config/timeout.go +++ b/conformance/utils/config/timeout.go @@ -51,9 +51,9 @@ type TimeoutConfig struct { // Max value for conformant implementation: None HTTPRouteMustHaveCondition time.Duration - // HTTPRouteMustHaveParents represents the maximum time for an HTTPRoute to have parents in status that match the expected parents. + // RouteMustHaveParents represents the maximum time for an xRoute to have parents in status that match the expected parents. // Max value for conformant implementation: None - HTTPRouteMustHaveParents time.Duration + RouteMustHaveParents time.Duration // ManifestFetchTimeout represents the maximum time for getting content from a https:// URL. // Max value for conformant implementation: None @@ -70,6 +70,11 @@ type TimeoutConfig struct { // RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper. // Max value for conformant implementation: None RequestTimeout time.Duration + + // RequiredConsecutiveSuccesses is the number of requests that must succeed in a row + // to consider a response "consistent" before making additional assertions on the response body. + // If this number is not reached within MaxTimeToConsistency, the test will fail. + RequiredConsecutiveSuccesses int } // DefaultTimeoutConfig populates a TimeoutConfig with the default values. @@ -83,11 +88,12 @@ func DefaultTimeoutConfig() TimeoutConfig { GWCMustBeAccepted: 180 * time.Second, HTTPRouteMustNotHaveParents: 60 * time.Second, HTTPRouteMustHaveCondition: 60 * time.Second, - HTTPRouteMustHaveParents: 60 * time.Second, + RouteMustHaveParents: 60 * time.Second, ManifestFetchTimeout: 10 * time.Second, MaxTimeToConsistency: 30 * time.Second, NamespacesMustBeReady: 300 * time.Second, RequestTimeout: 10 * time.Second, + RequiredConsecutiveSuccesses: 3, } } @@ -117,8 +123,8 @@ func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) { if timeoutConfig.HTTPRouteMustHaveCondition == 0 { timeoutConfig.HTTPRouteMustHaveCondition = defaultTimeoutConfig.HTTPRouteMustHaveCondition } - if timeoutConfig.HTTPRouteMustHaveParents == 0 { - timeoutConfig.HTTPRouteMustHaveParents = defaultTimeoutConfig.HTTPRouteMustHaveParents + if timeoutConfig.RouteMustHaveParents == 0 { + timeoutConfig.RouteMustHaveParents = defaultTimeoutConfig.RouteMustHaveParents } if timeoutConfig.ManifestFetchTimeout == 0 { timeoutConfig.ManifestFetchTimeout = defaultTimeoutConfig.ManifestFetchTimeout diff --git a/conformance/utils/http/http.go b/conformance/utils/http/http.go index 8b94aa63ad..93f72dabbd 100644 --- a/conformance/utils/http/http.go +++ b/conformance/utils/http/http.go @@ -82,12 +82,6 @@ type Response struct { AbsentHeaders []string } -// requiredConsecutiveSuccesses is the number of requests that must succeed in a row -// for MakeRequestAndExpectEventuallyConsistentResponse to consider the response "consistent" -// before making additional assertions on the response body. If this number is not reached within -// maxTimeToConsistency, the test will fail. -const requiredConsecutiveSuccesses = 3 - // MakeRequestAndExpectEventuallyConsistentResponse makes a request with the given parameters, // understanding that the request may fail for some amount of time. // @@ -96,6 +90,14 @@ const requiredConsecutiveSuccesses = 3 func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, expected ExpectedResponse) { t.Helper() + req := MakeRequest(t, &expected, gwAddr, "HTTP", "http") + + WaitForConsistentResponse(t, r, req, expected, timeoutConfig.RequiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency) +} + +func MakeRequest(t *testing.T, expected *ExpectedResponse, gwAddr, protocol, scheme string) roundtripper.Request { + t.Helper() + if expected.Request.Method == "" { expected.Request.Method = "GET" } @@ -104,15 +106,15 @@ func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripp expected.Response.StatusCode = 200 } - t.Logf("Making %s request to http://%s%s", expected.Request.Method, gwAddr, expected.Request.Path) + t.Logf("Making %s request to %s://%s%s", expected.Request.Method, scheme, gwAddr, expected.Request.Path) path, query, _ := strings.Cut(expected.Request.Path, "?") req := roundtripper.Request{ Method: expected.Request.Method, Host: expected.Request.Host, - URL: url.URL{Scheme: "http", Host: gwAddr, Path: path, RawQuery: query}, - Protocol: "HTTP", + URL: url.URL{Scheme: scheme, Host: gwAddr, Path: path, RawQuery: query}, + Protocol: protocol, Headers: map[string][]string{}, UnfollowRedirect: expected.Request.UnfollowRedirect, } @@ -129,12 +131,12 @@ func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripp } req.Headers["X-Echo-Set-Header"] = []string{strings.Join(backendSetHeaders, ",")} - WaitForConsistentResponse(t, r, req, expected, requiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency) + return req } -// awaitConvergence runs the given function until it returns 'true' `threshold` times in a row. +// AwaitConvergence runs the given function until it returns 'true' `threshold` times in a row. // Each failed attempt has a 1s delay; successful attempts have no delay. -func awaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Duration, fn func(elapsed time.Duration) bool) { +func AwaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Duration, fn func(elapsed time.Duration) bool) { successes := 0 attempts := 0 start := time.Now() @@ -162,7 +164,7 @@ func awaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Dur select { // Capture the overall timeout case <-to: - t.Fatalf("timeout while waiting after %d attempts, %d/%d sucessess", attempts, successes, threshold) + t.Fatalf("timeout while waiting after %d attempts, %d/%d successes", attempts, successes, threshold) // And the per-try delay case <-time.After(delay): } @@ -173,7 +175,7 @@ func awaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Dur // the expected response consistently. The provided threshold determines how many times in // a row this must occur to be considered "consistent". func WaitForConsistentResponse(t *testing.T, r roundtripper.RoundTripper, req roundtripper.Request, expected ExpectedResponse, threshold int, maxTimeToConsistency time.Duration) { - awaitConvergence(t, threshold, maxTimeToConsistency, func(elapsed time.Duration) bool { + AwaitConvergence(t, threshold, maxTimeToConsistency, func(elapsed time.Duration) bool { cReq, cRes, err := r.CaptureRoundTrip(req) if err != nil { t.Logf("Request failed, not ready yet: %v (after %v)", err.Error(), elapsed) @@ -312,7 +314,7 @@ func CompareRequest(req *roundtripper.Request, cReq *roundtripper.CapturedReques return nil } -// Get User-defined test case name or generate from expected response to a given request. +// GetTestCaseName gets the user-defined test case name or generates one from expected response to a given request. func (er *ExpectedResponse) GetTestCaseName(i int) string { // If TestCase name is provided then use that or else generate one. diff --git a/conformance/utils/kubernetes/certificate.go b/conformance/utils/kubernetes/certificate.go index 1077ca5681..1671cd12fd 100644 --- a/conformance/utils/kubernetes/certificate.go +++ b/conformance/utils/kubernetes/certificate.go @@ -71,7 +71,7 @@ func MustCreateSelfSignedCertSecret(t *testing.T, namespace, secretName string, return newSecret } -// generateRSACert generates a basic self signed certificate valir for a year +// generateRSACert generates a basic self signed certificate valid for a year func generateRSACert(host string, keyOut, certOut io.Writer) error { priv, err := rsa.GenerateKey(rand.Reader, rsaBits) if err != nil { diff --git a/conformance/utils/kubernetes/helpers.go b/conformance/utils/kubernetes/helpers.go index df0a1e571f..ca60ebeea2 100644 --- a/conformance/utils/kubernetes/helpers.go +++ b/conformance/utils/kubernetes/helpers.go @@ -28,12 +28,14 @@ import ( "time" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" "sigs.k8s.io/gateway-api/conformance/utils/config" ) @@ -150,8 +152,8 @@ func ConditionsHaveLatestObservedGeneration(obj metav1.Object, conditions []meta return errors.New(b.String()) } -// FilterStaleConditions returns the list of status condition whos observedGeneration does not -// match the objects metadata.Generation +// FilterStaleConditions returns the list of status condition whose observedGeneration does not +// match the object's metadata.Generation func FilterStaleConditions(obj metav1.Object, conditions []metav1.Condition) []metav1.Condition { stale := make([]metav1.Condition, 0, len(conditions)) for _, condition := range conditions { @@ -385,7 +387,7 @@ func HTTPRouteMustHaveParents(t *testing.T, client client.Client, timeoutConfig t.Helper() var actual []v1beta1.RouteParentStatus - waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.HTTPRouteMustHaveParents, func() (bool, error) { + waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.RouteMustHaveParents, func() (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -408,6 +410,33 @@ func HTTPRouteMustHaveParents(t *testing.T, client client.Client, timeoutConfig require.NoErrorf(t, waitErr, "error waiting for HTTPRoute to have parents matching expectations") } +// TLSRouteMustHaveParents waits for the specified TLSRoute to have parents +// in status that match the expected parents, and also returns the TLSRoute. +// This will cause the test to halt if the specified timeout is exceeded. +func TLSRouteMustHaveParents(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeName types.NamespacedName, parents []v1alpha2.RouteParentStatus, namespaceRequired bool) v1alpha2.TLSRoute { + t.Helper() + + var actual []v1beta1.RouteParentStatus + var route v1alpha2.TLSRoute + + waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.RouteMustHaveParents, func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err := client.Get(ctx, routeName, &route) + if err != nil { + return false, fmt.Errorf("error fetching TLSRoute: %w", err) + } + actual = route.Status.Parents + match := parentsForRouteMatch(t, routeName, parents, actual, namespaceRequired) + + return match, nil + }) + require.NoErrorf(t, waitErr, "error waiting for TLSRoute to have parents matching expectations") + + return route +} + func parentsForRouteMatch(t *testing.T, routeName types.NamespacedName, expected, actual []v1beta1.RouteParentStatus, namespaceRequired bool) bool { t.Helper() @@ -479,7 +508,7 @@ func GatewayStatusMustHaveListeners(t *testing.T, client client.Client, timeoutC require.NoErrorf(t, waitErr, "error waiting for Gateway status to have listeners matching expectations") } -// HTTPRouteMustHaveConditions checks that the supplied HTTPRoute has the supplied Condition, +// HTTPRouteMustHaveCondition checks that the supplied HTTPRoute has the supplied Condition, // halting after the specified timeout is exceeded. func HTTPRouteMustHaveCondition(t *testing.T, client client.Client, timeoutConfig config.TimeoutConfig, routeNN types.NamespacedName, gwNN types.NamespacedName, condition metav1.Condition) { t.Helper() @@ -523,6 +552,54 @@ func parentRefToString(p v1beta1.ParentReference) string { return string(p.Name) } +// GatewayAndTLSRoutesMustBeAccepted waits until the specified Gateway has an IP +// address assigned to it and the TLSRoute has a ParentRef referring to the +// Gateway. The test will fail if these conditions are not met before the +// timeouts. +func GatewayAndTLSRoutesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, controllerName string, gw GatewayRef, routeNNs ...types.NamespacedName) (string, []v1beta1.Hostname) { + t.Helper() + + var hostnames []v1beta1.Hostname + + gwAddr, err := WaitForGatewayAddress(t, c, timeoutConfig, gw.NamespacedName) + require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned") + + ns := v1beta1.Namespace(gw.Namespace) + kind := v1beta1.Kind("Gateway") + + for _, routeNN := range routeNNs { + namespaceRequired := true + if routeNN.Namespace == gw.Namespace { + namespaceRequired = false + } + + var parents []v1beta1.RouteParentStatus + for _, listener := range gw.listenerNames { + parents = append(parents, v1beta1.RouteParentStatus{ + ParentRef: v1beta1.ParentReference{ + Group: (*v1beta1.Group)(&v1beta1.GroupVersion.Group), + Kind: &kind, + Name: v1beta1.ObjectName(gw.Name), + Namespace: &ns, + SectionName: listener, + }, + ControllerName: v1beta1.GatewayController(controllerName), + Conditions: []metav1.Condition{ + { + Type: string(v1beta1.RouteConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(v1beta1.RouteReasonAccepted), + }, + }, + }) + } + route := TLSRouteMustHaveParents(t, c, timeoutConfig, routeNN, parents, namespaceRequired) + hostnames = route.Spec.Hostnames + } + + return gwAddr, hostnames +} + // TODO(mikemorris): this and parentsMatch could possibly be rewritten as a generic function? func listenersMatch(t *testing.T, expected, actual []v1beta1.ListenerStatus) bool { t.Helper() @@ -558,6 +635,8 @@ func listenersMatch(t *testing.T, expected, actual []v1beta1.ListenerStatus) boo } func conditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool { + t.Helper() + if len(actual) < len(expected) { t.Logf("Expected more conditions to be present") return false @@ -575,6 +654,8 @@ func conditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool { // findConditionInList finds a condition in a list of Conditions, checking // the Name, Value, and Reason. If an empty reason is passed, any Reason will match. func findConditionInList(t *testing.T, conditions []metav1.Condition, condName, expectedStatus, expectedReason string) bool { + t.Helper() + for _, cond := range conditions { if cond.Type == condName { if cond.Status == metav1.ConditionStatus(expectedStatus) { @@ -589,11 +670,13 @@ func findConditionInList(t *testing.T, conditions []metav1.Condition, condName, } } - t.Logf("%s was not in conditions list", condName) + t.Logf("%s was not in conditions list [%v]", condName, conditions) return false } func findPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool { + t.Helper() + for _, cond := range conditions { if cond.Type == v1.PodConditionType(condName) { if cond.Status == v1.ConditionStatus(condValue) { diff --git a/conformance/utils/roundtripper/roundtripper.go b/conformance/utils/roundtripper/roundtripper.go index 284bac5079..2b3a7e1b90 100644 --- a/conformance/utils/roundtripper/roundtripper.go +++ b/conformance/utils/roundtripper/roundtripper.go @@ -18,9 +18,12 @@ package roundtripper import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" - "io/ioutil" + "io" + iou "io/ioutil" "net/http" "net/http/httputil" "net/url" @@ -43,6 +46,9 @@ type Request struct { Method string Headers map[string][]string UnfollowRedirect bool + CertPem []byte + KeyPem []byte + Server string } // CapturedRequest contains request metadata captured from an echoserver @@ -97,6 +103,36 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques } } + // Setup TLS transport if there are CertPem, KeyPem, and Server in the request + if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { + // Create a certificate from the provided cert and key + cert, err := tls.X509KeyPair(request.CertPem, request.KeyPem) + if err != nil { + return nil, nil, fmt.Errorf("unexpected error creating cert: %w", err) + } + + // Add the provided cert as a trusted CA + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(request.CertPem) { + return nil, nil, fmt.Errorf("unexpected error adding trusted CA: %w", err) + } + + if request.Server == "" { + return nil, nil, fmt.Errorf("unexpected error, server name required for TLS") + } + + // Create the Transport for this provided host, cert, and trusted CA + client.Transport = &http.Transport{ + // Disable G402: TLS MinVersion too low. (gosec) + // #nosec G402 + TLSClientConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + ServerName: request.Server, + RootCAs: certPool, + }, + } + } + method := "GET" if request.Method != "" { method = request.Method @@ -132,7 +168,9 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques if err != nil { return nil, nil, err } - defer resp.Body.Close() + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) if d.Debug { var dump []byte @@ -144,7 +182,7 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques fmt.Printf("Received Response:\n%s\n\n", formatDump(dump, "< ")) } - body, _ := ioutil.ReadAll(resp.Body) + body, _ := iou.ReadAll(resp.Body) // we cannot assume the response is JSON if resp.Header.Get("Content-type") == "application/json" { diff --git a/conformance/utils/suite/suite.go b/conformance/utils/suite/suite.go index 6454a7c2db..095ba011ba 100644 --- a/conformance/utils/suite/suite.go +++ b/conformance/utils/suite/suite.go @@ -174,6 +174,8 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) { suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) secret = kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-infra", "tls-validity-checks-certificate", []string{"*"}) suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) + secret = kubernetes.MustCreateSelfSignedCertSecret(t, "gateway-conformance-infra", "tls-passthrough-checks-certificate", []string{"abc.example.com"}) + suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) t.Logf("Test Setup: Ensuring Gateways and Pods from base manifests are ready") namespaces := []string{ diff --git a/conformance/utils/tls/tls.go b/conformance/utils/tls/tls.go new file mode 100644 index 0000000000..16e2326726 --- /dev/null +++ b/conformance/utils/tls/tls.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tls + +import ( + "testing" + "time" + + "sigs.k8s.io/gateway-api/conformance/utils/config" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" +) + +// MakeTLSRequestAndExpectEventuallyConsistentResponse makes a request with the given parameters, +// understanding that the request may fail for some amount of time. +// +// Once the request succeeds consistently with the response having the expected status code, make +// additional assertions on the response body using the provided ExpectedResponse. +func MakeTLSRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, cPem, keyPem []byte, server string, expected http.ExpectedResponse) { + t.Helper() + + req := http.MakeRequest(t, &expected, gwAddr, "HTTPS", "https") + + WaitForConsistentTLSResponse(t, r, req, expected, timeoutConfig.RequiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency, cPem, keyPem, server) +} + +// WaitForConsistentTLSResponse - repeats the provided request until it completes with a response having +// the expected response consistently. The provided threshold determines how many times in +// a row this must occur to be considered "consistent". +func WaitForConsistentTLSResponse(t *testing.T, r roundtripper.RoundTripper, req roundtripper.Request, expected http.ExpectedResponse, threshold int, maxTimeToConsistency time.Duration, cPem, keyPem []byte, server string) { + http.AwaitConvergence(t, threshold, maxTimeToConsistency, func(elapsed time.Duration) bool { + req.KeyPem = keyPem + req.CertPem = cPem + req.Server = server + + cReq, cRes, err := r.CaptureRoundTrip(req) + if err != nil { + t.Logf("Request failed, not ready yet: %v (after %v)", err.Error(), elapsed) + return false + } + + if err := http.CompareRequest(&req, cReq, cRes, expected); err != nil { + t.Logf("Response expectation failed for request: %v not ready yet: %v (after %v)", req, err, elapsed) + return false + } + + return true + }) + t.Logf("Request passed") +} diff --git a/site-src/concepts/conformance.md b/site-src/concepts/conformance.md index 5d4ca78a39..8f43f433ab 100644 --- a/site-src/concepts/conformance.md +++ b/site-src/concepts/conformance.md @@ -74,15 +74,27 @@ capabilities. ### Running Tests -By default, conformance tests will expect a `gateway-conformance` GatewayClass -to be installed in the cluster and tests will be run against that. A different -class can be specified with the `--gateway-class` flag along with the -corresponding test command. For example: +By default, conformance tests will expect a GatewayClass named `gateway-conformance` +to be installed in the cluster, and tests will be run against that. Most often, +you'll use a different class, which can be specified with the `-gateway-class` flag along with the +corresponding test command. Check your instance for the `gateway-class` name to use. ```shell -go test ./conformance --gateway-class my-class +go test ./conformance/... -args -gateway-class=my-gateway-class ``` +Other useful flags may be found in +[conformance flags](https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/utils/flags/flags.go). +For example, if you'd like to examine the objects in Kubernetes after your test runs, you can pass a flag to +suppress cleanup: +```shell +go test ./conformance/... -args -gateway-class=istio -cleanup-base-resources=false +``` +If you'd like to run a single test instead of the entire conformance suite, find your test name +`(suite.ConformanceTest.ShortName)` and use it like this: +```shell +go test ./conformance/... --run TestConformance/YOURTESTNAME --gateway-class=istio +``` ## Contributing to Conformance Many implementations run conformance tests as part of their full e2e test suite.