Skip to content

Commit

Permalink
Add sslpassthrough tests (#9457)
Browse files Browse the repository at this point in the history
  • Loading branch information
rikatz authored Dec 28, 2022
1 parent a8f4f29 commit fe2bf5c
Show file tree
Hide file tree
Showing 6 changed files with 231 additions and 28 deletions.
8 changes: 4 additions & 4 deletions build/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@

GO_BUILD_CMD="go build"

if [ -n "$DEBUG" ]; then
set -x
GO_BUILD_CMD="go build -v"
fi
#if [ -n "$DEBUG" ]; then
# set -x
# GO_BUILD_CMD="go build -v"
#fi

set -o errexit
set -o nounset
Expand Down
16 changes: 7 additions & 9 deletions build/run-in-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,23 +65,21 @@ fi

USER=${USER:-nobody}

echo "..printing env & other vars to stdout"
echo "HOSTNAME=`hostname`"
uname -a
env
echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED"
echo "done..printing env & other vars to stdout"
#echo "..printing env & other vars to stdout"
#echo "HOSTNAME=`hostname`"
#uname -a
#env
#echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED"
#echo "done..printing env & other vars to stdout"

if [[ "$DOCKER_IN_DOCKER_ENABLED" == "true" ]]; then
echo "..reached DIND check TRUE block, inside run-in-docker.sh"
echo "FLAGS=$FLAGS"
go env
set -x
#go env
go install -mod=mod github.com/onsi/ginkgo/v2/[email protected]
find / -type f -name ginkgo 2>/dev/null
which ginkgo
/bin/bash -c "${FLAGS}"
set +x
else
echo "Reached DIND check ELSE block, inside run-in-docker.sh"
docker run \
Expand Down
8 changes: 3 additions & 5 deletions pkg/tcpproxy/tcp.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func (p *TCPProxy) Handle(conn net.Conn) {
}

hostPort := net.JoinHostPort(proxy.IP, fmt.Sprintf("%v", proxy.Port))
klog.V(4).InfoS("passing to", "hostport", hostPort)
clientConn, err := net.Dial("tcp", hostPort)
if err != nil {
klog.V(4).ErrorS(err, "error dialing proxy", "ip", proxy.IP, "port", proxy.Port, "hostname", proxy.Hostname)
Expand All @@ -99,7 +100,7 @@ func (p *TCPProxy) Handle(conn net.Conn) {
}
proxyProtocolHeader := fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, remoteAddr.IP.String(), localAddr.IP.String(), remoteAddr.Port, localAddr.Port)
klog.V(4).InfoS("Writing Proxy Protocol", "header", proxyProtocolHeader)
_, err = fmt.Fprintf(clientConn, proxyProtocolHeader)
_, err = fmt.Fprint(clientConn, proxyProtocolHeader)
}
if err != nil {
klog.ErrorS(err, "Error writing Proxy Protocol header")
Expand All @@ -126,8 +127,5 @@ func pipe(client, server net.Conn) {
go doCopy(server, client, cancel)
go doCopy(client, server, cancel)

select {
case <-cancel:
return
}
<-cancel
}
33 changes: 23 additions & 10 deletions test/e2e/framework/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,10 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
}

deployment := newDeployment(options.name, options.namespace, "registry.k8s.io/ingress-nginx/e2e-test-echo@sha256:778ac6d1188c8de8ecabeddd3c37b72c8adc8c712bad2bd7a81fb23a3514934c", 80, int32(options.replicas),
nil,
nil, nil, nil,
[]corev1.VolumeMount{},
[]corev1.Volume{},
true,
)

f.EnsureDeployment(deployment)
Expand Down Expand Up @@ -183,7 +184,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool)
assert.Nil(ginkgo.GinkgoT(), err, "creating configmap")

deployment := newDeployment(name, f.Namespace, f.GetNginxBaseImage(), 80, 1,
nil,
nil, nil, nil,
[]corev1.VolumeMount{
{
Name: name,
Expand All @@ -203,7 +204,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool)
},
},
},
},
}, true,
)

f.EnsureDeployment(deployment)
Expand Down Expand Up @@ -334,8 +335,8 @@ func (f *Framework) NewGRPCBinDeployment() {
assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
}

func newDeployment(name, namespace, image string, port int32, replicas int32, command []string,
volumeMounts []corev1.VolumeMount, volumes []corev1.Volume) *appsv1.Deployment {
func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar,
volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) *appsv1.Deployment {
probe := &corev1.Probe{
InitialDelaySeconds: 2,
PeriodSeconds: 1,
Expand Down Expand Up @@ -381,9 +382,7 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co
ContainerPort: port,
},
},
ReadinessProbe: probe,
LivenessProbe: probe,
VolumeMounts: volumeMounts,
VolumeMounts: volumeMounts,
},
},
Volumes: volumes,
Expand All @@ -392,10 +391,20 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co
},
}

if setProbe {
d.Spec.Template.Spec.Containers[0].ReadinessProbe = probe
d.Spec.Template.Spec.Containers[0].LivenessProbe = probe
}
if len(command) > 0 {
d.Spec.Template.Spec.Containers[0].Command = command
}

if len(args) > 0 {
d.Spec.Template.Spec.Containers[0].Args = args
}
if len(env) > 0 {
d.Spec.Template.Spec.Containers[0].Env = env
}
return d
}

Expand All @@ -404,9 +413,13 @@ func (f *Framework) NewHttpbinDeployment() {
f.NewDeployment(HTTPBinService, "registry.k8s.io/ingress-nginx/e2e-test-httpbin@sha256:c6372ef57a775b95f18e19d4c735a9819f2e7bb4641e5e3f27287d831dfeb7e8", 80, 1)
}

// NewDeployment creates a new deployment in a particular namespace.
func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) {
deployment := newDeployment(name, f.Namespace, image, port, replicas, nil, nil, nil)
f.NewDeploymentWithOpts(name, image, port, replicas, nil, nil, nil, nil, nil, true)
}

// NewDeployment creates a new deployment in a particular namespace.
func (f *Framework) NewDeploymentWithOpts(name, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar, volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) {
deployment := newDeployment(name, f.Namespace, image, port, replicas, command, args, env, volumeMounts, volumes, setProbe)

f.EnsureDeployment(deployment)

Expand Down
29 changes: 29 additions & 0 deletions test/e2e/framework/httpexpect/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ limitations under the License.
package httpexpect

import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
Expand Down Expand Up @@ -71,6 +73,33 @@ func (h *HTTPRequest) DoRequest(method, rpath string) *HTTPRequest {
return h
}

// ForceResolve forces the test resolver to point to a specific endpoint
func (h *HTTPRequest) ForceResolve(ip string, port uint16) *HTTPRequest {
addr := net.ParseIP(ip)
if addr == nil {
h.chain.fail(fmt.Sprintf("invalid ip address: %s", ip))
return h
}
dialer := &net.Dialer{
Timeout: h.client.Timeout,
KeepAlive: h.client.Timeout,
DualStack: true,
}
resolveAddr := fmt.Sprintf("%s:%d", ip, int(port))

oldTransport, ok := h.client.Transport.(*http.Transport)
if !ok {
h.chain.fail("invalid old transport address")
return h
}
newTransport := oldTransport.Clone()
newTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
return dialer.DialContext(ctx, network, resolveAddr)
}
h.client.Transport = newTransport
return h
}

// Expect executes the request and returns an HTTP response.
func (h *HTTPRequest) Expect() *HTTPResponse {
if h.query != nil {
Expand Down
165 changes: 165 additions & 0 deletions test/e2e/settings/ssl_passthrough.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package settings

import (
"context"
"crypto/tls"
"fmt"
"net/http"
"strings"

"github.com/onsi/ginkgo/v2"
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() {
f := framework.NewDefaultFramework("ssl-passthrough")

ginkgo.BeforeEach(func() {
err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
args := deployment.Spec.Template.Spec.Containers[0].Args
args = append(args, "--enable-ssl-passthrough")
deployment.Spec.Template.Spec.Containers[0].Args = args
_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
return err
})
assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")

f.WaitForNginxServer("_",
func(server string) bool {
return strings.Contains(server, "listen 442")
})
})

ginkgo.Describe("With enable-ssl-passthrough enabled", func() {
ginkgo.It("should enable ssl-passthrough-proxy-port on a different port", func() {

err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
args := deployment.Spec.Template.Spec.Containers[0].Args
args = append(args, "--ssl-passthrough-proxy-port=1442")
deployment.Spec.Template.Spec.Containers[0].Args = args
_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
return err
})
assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")

f.WaitForNginxServer("_",
func(server string) bool {
return strings.Contains(server, "listen 1442")
})

f.HTTPTestClient().
GET("/").
WithHeader("Host", "something").
Expect().
Status(http.StatusNotFound)
})

ginkgo.It("should pass unknown traffic to default backend and handle known traffic", func() {

host := "testpassthrough.com"
echoName := "echopass"

/* Even with enable-ssl-passthrough enabled, only annotated ingresses may receive the trafic */
annotations := map[string]string{
"nginx.ingress.kubernetes.io/ssl-passthrough": "true",
}

ingressDef := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, echoName, 80, annotations)
tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
ingressDef.Spec.TLS[0].Hosts,
ingressDef.Spec.TLS[0].SecretName,
ingressDef.Namespace)

volumeMount := []corev1.VolumeMount{
{
Name: "certs",
ReadOnly: true,
MountPath: "/certs",
},
}
volume := []corev1.Volume{
{
Name: "certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: ingressDef.Spec.TLS[0].SecretName,
},
},
},
}
envs := []corev1.EnvVar{
{
Name: "HTTPBUN_SSL_CERT",
Value: "/certs/tls.crt",
},
{
Name: "HTTPBUN_SSL_KEY",
Value: "/certs/tls.key",
},
}
f.NewDeploymentWithOpts("echopass", "ghcr.io/sharat87/httpbun:latest", 80, 1, nil, nil, envs, volumeMount, volume, false)

f.EnsureIngress(ingressDef)

assert.Nil(ginkgo.GinkgoT(), err)
framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig)

f.WaitForNginxServer(host,
func(server string) bool {
return strings.Contains(server, "listen 442")
})

/* This one should not receive traffic as it does not contain passthrough annotation */
hostBad := "noannotationnopassthrough.com"
ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, "/", hostBad, []string{hostBad}, f.Namespace, echoName, 80, nil))
tlsConfigBad, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
ingBad.Spec.TLS[0].Hosts,
ingBad.Spec.TLS[0].SecretName,
ingBad.Namespace)
assert.Nil(ginkgo.GinkgoT(), err)
framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfigBad)

f.WaitForNginxServer(hostBad,
func(server string) bool {
return strings.Contains(server, "listen 442")
})

f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}).
GET("/").
WithURL(fmt.Sprintf("https://%s:443", host)).
ForceResolve(f.GetNginxIP(), 443).
Expect().
Status(http.StatusOK)

f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: hostBad, InsecureSkipVerify: true}).
GET("/").
WithURL(fmt.Sprintf("https://%s:443", hostBad)).
ForceResolve(f.GetNginxIP(), 443).
Expect().
Status(http.StatusNotFound)

})
})
})

0 comments on commit fe2bf5c

Please sign in to comment.