From fe2bf5cbdf614fe40348c7b2484ead52a7216f5a Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Wed, 28 Dec 2022 17:59:27 -0300 Subject: [PATCH] Add sslpassthrough tests (#9457) --- build/build.sh | 8 +- build/run-in-docker.sh | 16 +-- pkg/tcpproxy/tcp.go | 8 +- test/e2e/framework/deployment.go | 33 +++-- test/e2e/framework/httpexpect/request.go | 29 ++++ test/e2e/settings/ssl_passthrough.go | 165 +++++++++++++++++++++++ 6 files changed, 231 insertions(+), 28 deletions(-) create mode 100644 test/e2e/settings/ssl_passthrough.go diff --git a/build/build.sh b/build/build.sh index f2f1ec2eb1..1ae505d910 100755 --- a/build/build.sh +++ b/build/build.sh @@ -16,10 +16,10 @@ GO_BUILD_CMD="go build" -if [ -n "$DEBUG" ]; then - set -x - GO_BUILD_CMD="go build -v" -fi +#if [ -n "$DEBUG" ]; then +# set -x +# GO_BUILD_CMD="go build -v" +#fi set -o errexit set -o nounset diff --git a/build/run-in-docker.sh b/build/run-in-docker.sh index 511c5475f9..d41464e2ec 100755 --- a/build/run-in-docker.sh +++ b/build/run-in-docker.sh @@ -65,23 +65,21 @@ fi USER=${USER:-nobody} -echo "..printing env & other vars to stdout" -echo "HOSTNAME=`hostname`" -uname -a -env -echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED" -echo "done..printing env & other vars to stdout" +#echo "..printing env & other vars to stdout" +#echo "HOSTNAME=`hostname`" +#uname -a +#env +#echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED" +#echo "done..printing env & other vars to stdout" if [[ "$DOCKER_IN_DOCKER_ENABLED" == "true" ]]; then echo "..reached DIND check TRUE block, inside run-in-docker.sh" echo "FLAGS=$FLAGS" - go env - set -x + #go env go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.6.1 find / -type f -name ginkgo 2>/dev/null which ginkgo /bin/bash -c "${FLAGS}" - set +x else echo "Reached DIND check ELSE block, inside run-in-docker.sh" docker run \ diff --git a/pkg/tcpproxy/tcp.go b/pkg/tcpproxy/tcp.go index 7bbff80b44..4c34e1f7b2 100644 --- a/pkg/tcpproxy/tcp.go +++ b/pkg/tcpproxy/tcp.go @@ -80,6 +80,7 @@ func (p *TCPProxy) Handle(conn net.Conn) { } hostPort := net.JoinHostPort(proxy.IP, fmt.Sprintf("%v", proxy.Port)) + klog.V(4).InfoS("passing to", "hostport", hostPort) clientConn, err := net.Dial("tcp", hostPort) if err != nil { klog.V(4).ErrorS(err, "error dialing proxy", "ip", proxy.IP, "port", proxy.Port, "hostname", proxy.Hostname) @@ -99,7 +100,7 @@ func (p *TCPProxy) Handle(conn net.Conn) { } proxyProtocolHeader := fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, remoteAddr.IP.String(), localAddr.IP.String(), remoteAddr.Port, localAddr.Port) klog.V(4).InfoS("Writing Proxy Protocol", "header", proxyProtocolHeader) - _, err = fmt.Fprintf(clientConn, proxyProtocolHeader) + _, err = fmt.Fprint(clientConn, proxyProtocolHeader) } if err != nil { klog.ErrorS(err, "Error writing Proxy Protocol header") @@ -126,8 +127,5 @@ func pipe(client, server net.Conn) { go doCopy(server, client, cancel) go doCopy(client, server, cancel) - select { - case <-cancel: - return - } + <-cancel } diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index cb6ef9accb..3cfe8f3604 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -85,9 +85,10 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) { } deployment := newDeployment(options.name, options.namespace, "registry.k8s.io/ingress-nginx/e2e-test-echo@sha256:778ac6d1188c8de8ecabeddd3c37b72c8adc8c712bad2bd7a81fb23a3514934c", 80, int32(options.replicas), - nil, + nil, nil, nil, []corev1.VolumeMount{}, []corev1.Volume{}, + true, ) f.EnsureDeployment(deployment) @@ -183,7 +184,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool) assert.Nil(ginkgo.GinkgoT(), err, "creating configmap") deployment := newDeployment(name, f.Namespace, f.GetNginxBaseImage(), 80, 1, - nil, + nil, nil, nil, []corev1.VolumeMount{ { Name: name, @@ -203,7 +204,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool) }, }, }, - }, + }, true, ) f.EnsureDeployment(deployment) @@ -334,8 +335,8 @@ func (f *Framework) NewGRPCBinDeployment() { assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") } -func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, - volumeMounts []corev1.VolumeMount, volumes []corev1.Volume) *appsv1.Deployment { +func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar, + volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) *appsv1.Deployment { probe := &corev1.Probe{ InitialDelaySeconds: 2, PeriodSeconds: 1, @@ -381,9 +382,7 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co ContainerPort: port, }, }, - ReadinessProbe: probe, - LivenessProbe: probe, - VolumeMounts: volumeMounts, + VolumeMounts: volumeMounts, }, }, Volumes: volumes, @@ -392,10 +391,20 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co }, } + if setProbe { + d.Spec.Template.Spec.Containers[0].ReadinessProbe = probe + d.Spec.Template.Spec.Containers[0].LivenessProbe = probe + } if len(command) > 0 { d.Spec.Template.Spec.Containers[0].Command = command } + if len(args) > 0 { + d.Spec.Template.Spec.Containers[0].Args = args + } + if len(env) > 0 { + d.Spec.Template.Spec.Containers[0].Env = env + } return d } @@ -404,9 +413,13 @@ func (f *Framework) NewHttpbinDeployment() { f.NewDeployment(HTTPBinService, "registry.k8s.io/ingress-nginx/e2e-test-httpbin@sha256:c6372ef57a775b95f18e19d4c735a9819f2e7bb4641e5e3f27287d831dfeb7e8", 80, 1) } -// NewDeployment creates a new deployment in a particular namespace. func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { - deployment := newDeployment(name, f.Namespace, image, port, replicas, nil, nil, nil) + f.NewDeploymentWithOpts(name, image, port, replicas, nil, nil, nil, nil, nil, true) +} + +// NewDeployment creates a new deployment in a particular namespace. +func (f *Framework) NewDeploymentWithOpts(name, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar, volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) { + deployment := newDeployment(name, f.Namespace, image, port, replicas, command, args, env, volumeMounts, volumes, setProbe) f.EnsureDeployment(deployment) diff --git a/test/e2e/framework/httpexpect/request.go b/test/e2e/framework/httpexpect/request.go index 335e3931ed..d8edb42ceb 100644 --- a/test/e2e/framework/httpexpect/request.go +++ b/test/e2e/framework/httpexpect/request.go @@ -17,8 +17,10 @@ limitations under the License. package httpexpect import ( + "context" "fmt" "io" + "net" "net/http" "net/url" "path" @@ -71,6 +73,33 @@ func (h *HTTPRequest) DoRequest(method, rpath string) *HTTPRequest { return h } +// ForceResolve forces the test resolver to point to a specific endpoint +func (h *HTTPRequest) ForceResolve(ip string, port uint16) *HTTPRequest { + addr := net.ParseIP(ip) + if addr == nil { + h.chain.fail(fmt.Sprintf("invalid ip address: %s", ip)) + return h + } + dialer := &net.Dialer{ + Timeout: h.client.Timeout, + KeepAlive: h.client.Timeout, + DualStack: true, + } + resolveAddr := fmt.Sprintf("%s:%d", ip, int(port)) + + oldTransport, ok := h.client.Transport.(*http.Transport) + if !ok { + h.chain.fail("invalid old transport address") + return h + } + newTransport := oldTransport.Clone() + newTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return dialer.DialContext(ctx, network, resolveAddr) + } + h.client.Transport = newTransport + return h +} + // Expect executes the request and returns an HTTP response. func (h *HTTPRequest) Expect() *HTTPResponse { if h.query != nil { diff --git a/test/e2e/settings/ssl_passthrough.go b/test/e2e/settings/ssl_passthrough.go new file mode 100644 index 0000000000..77a3c990eb --- /dev/null +++ b/test/e2e/settings/ssl_passthrough.go @@ -0,0 +1,165 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { + f := framework.NewDefaultFramework("ssl-passthrough") + + ginkgo.BeforeEach(func() { + err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--enable-ssl-passthrough") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") + + f.WaitForNginxServer("_", + func(server string) bool { + return strings.Contains(server, "listen 442") + }) + }) + + ginkgo.Describe("With enable-ssl-passthrough enabled", func() { + ginkgo.It("should enable ssl-passthrough-proxy-port on a different port", func() { + + err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--ssl-passthrough-proxy-port=1442") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") + + f.WaitForNginxServer("_", + func(server string) bool { + return strings.Contains(server, "listen 1442") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", "something"). + Expect(). + Status(http.StatusNotFound) + }) + + ginkgo.It("should pass unknown traffic to default backend and handle known traffic", func() { + + host := "testpassthrough.com" + echoName := "echopass" + + /* Even with enable-ssl-passthrough enabled, only annotated ingresses may receive the trafic */ + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + } + + ingressDef := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, echoName, 80, annotations) + tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet, + ingressDef.Spec.TLS[0].Hosts, + ingressDef.Spec.TLS[0].SecretName, + ingressDef.Namespace) + + volumeMount := []corev1.VolumeMount{ + { + Name: "certs", + ReadOnly: true, + MountPath: "/certs", + }, + } + volume := []corev1.Volume{ + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: ingressDef.Spec.TLS[0].SecretName, + }, + }, + }, + } + envs := []corev1.EnvVar{ + { + Name: "HTTPBUN_SSL_CERT", + Value: "/certs/tls.crt", + }, + { + Name: "HTTPBUN_SSL_KEY", + Value: "/certs/tls.key", + }, + } + f.NewDeploymentWithOpts("echopass", "ghcr.io/sharat87/httpbun:latest", 80, 1, nil, nil, envs, volumeMount, volume, false) + + f.EnsureIngress(ingressDef) + + assert.Nil(ginkgo.GinkgoT(), err) + framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "listen 442") + }) + + /* This one should not receive traffic as it does not contain passthrough annotation */ + hostBad := "noannotationnopassthrough.com" + ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, "/", hostBad, []string{hostBad}, f.Namespace, echoName, 80, nil)) + tlsConfigBad, err := framework.CreateIngressTLSSecret(f.KubeClientSet, + ingBad.Spec.TLS[0].Hosts, + ingBad.Spec.TLS[0].SecretName, + ingBad.Namespace) + assert.Nil(ginkgo.GinkgoT(), err) + framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfigBad) + + f.WaitForNginxServer(hostBad, + func(server string) bool { + return strings.Contains(server, "listen 442") + }) + + f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). + GET("/"). + WithURL(fmt.Sprintf("https://%s:443", host)). + ForceResolve(f.GetNginxIP(), 443). + Expect(). + Status(http.StatusOK) + + f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: hostBad, InsecureSkipVerify: true}). + GET("/"). + WithURL(fmt.Sprintf("https://%s:443", hostBad)). + ForceResolve(f.GetNginxIP(), 443). + Expect(). + Status(http.StatusNotFound) + + }) + }) +})