From 32df03179554c41643334898322f3da535afbf9c Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:42:08 +0100 Subject: [PATCH 1/4] add integration test that directly invokes credential helper --- MODULE.bazel | 1 + go.mod | 3 +- go.sum | 6 +- integration/BUILD.bazel | 15 +++ integration/framework_test.go | 193 ++++++++++++++++++++++++++++++++++ integration/oci_test.go | 28 +++++ 6 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 integration/BUILD.bazel create mode 100644 integration/framework_test.go create mode 100644 integration/oci_test.go diff --git a/MODULE.bazel b/MODULE.bazel index 11ab63c..276f58a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,6 +16,7 @@ use_repo( "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_credentials", + "com_github_bazelbuild_rules_go", "com_github_stretchr_testify", "com_github_zalando_go_keyring", "io_k8s_sigs_yaml", diff --git a/go.mod b/go.mod index 2b52bfd..d771a5b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.28.5 github.com/aws/aws-sdk-go-v2/credentials v1.17.46 + github.com/bazelbuild/rules_go v0.53.0 github.com/stretchr/testify v1.10.0 github.com/zalando/go-keyring v0.2.6 golang.org/x/oauth2 v0.24.0 @@ -29,6 +30,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.28.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 24a9e6f..0933f79 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLb github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/bazelbuild/rules_go v0.53.0 h1:u160DT+RRb+Xb2aSO4piN8xhs4aZvWz2UDXCq48F4ao= +github.com/bazelbuild/rules_go v0.53.0/go.mod h1:xB1jfsYHWlnZyPPxzlOSst4q2ZAwS251Mp9Iw6TPuBc= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -48,8 +50,8 @@ github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8u github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/integration/BUILD.bazel b/integration/BUILD.bazel new file mode 100644 index 0000000..27d8f90 --- /dev/null +++ b/integration/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "integration_test", + srcs = [ + "framework_test.go", + "oci_test.go", + ], + data = ["//:tweag-credential-helper"], + env = {"CREDENTIAL_HELPER_LOGGING": "debug"}, + deps = [ + "//api", + "@rules_go//go/runfiles", + ], +) diff --git a/integration/framework_test.go b/integration/framework_test.go new file mode 100644 index 0000000..77afb14 --- /dev/null +++ b/integration/framework_test.go @@ -0,0 +1,193 @@ +package integration + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "os" + "os/exec" + "testing" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/tweag/credential-helper/api" +) + +type response struct{} + +type TestBuilder struct { + credentialHelperBinary string + t *testing.T +} + +func (builder *TestBuilder) WithHelperBinary(credentialHelperBinary string) *TestBuilder { + builder.credentialHelperBinary = credentialHelperBinary + return builder +} + +func (builder *TestBuilder) WithDefaultHelperBinary() *TestBuilder { + if os.Getenv("BAZEL_TEST") == "1" { + runfileLocation, err := runfiles.Rlocation("_main/tweag-credential-helper_/tweag-credential-helper") + if err != nil { + panic(err) + } + builder.credentialHelperBinary = runfileLocation + } else { + builder.credentialHelperBinary = "tweag-credential-helper" + } + return builder +} + +func (builder *TestBuilder) WithT(t *testing.T) *TestBuilder { + builder.t = t + return builder +} + +func (builder *TestBuilder) Build() *CredentialHelperTest { + return &CredentialHelperTest{ + credentialHelperBinary: builder.credentialHelperBinary, + t: builder.t, + } +} + +type CredentialHelperTest struct { + credentialHelperBinary string + t *testing.T +} + +func (test *CredentialHelperTest) invoke(ctx context.Context, stdin []byte, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, test.credentialHelperBinary, args...) + cmd.Stdin = bytes.NewReader(stdin) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + return out, err +} + +func (test *CredentialHelperTest) getCommand(ctx context.Context, uri string) (api.GetCredentialsResponse, error) { + stdin, err := json.Marshal(api.GetCredentialsRequest{URI: uri}) + if err != nil { + return api.GetCredentialsResponse{}, err + } + + var response api.GetCredentialsResponse + stdout, err := test.invoke(ctx, stdin, "get") + if err != nil { + return api.GetCredentialsResponse{}, err + } + if err := json.Unmarshal(stdout, &response); err != nil { + return api.GetCredentialsResponse{}, err + } + return response, nil +} + +func (test *CredentialHelperTest) Fetch(ctx context.Context, uri string) (*FetchResult, error) { + helperResp, err := test.getCommand(ctx, uri) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, http.NoBody) + if err != nil { + return nil, err + } + + for key, values := range helperResp.Headers { + for _, value := range values { + req.Header.Add(key, value) + } + } + + client := http.Client{} + + httpResp, err := client.Do(req) + if err != nil { + return nil, err + } + + defer httpResp.Body.Close() + + body, err := io.ReadAll(httpResp.Body) + if err != nil { + return nil, err + } + + return &FetchResult{ + helperResp: &helperResp, + httpResp: httpResp, + body: body, + t: test.t, + }, nil +} + +type FetchResult struct { + helperResp *api.GetCredentialsResponse + httpResp *http.Response + body []byte + t *testing.T +} + +func (result *FetchResult) ExpectStatusCode(expected int) { + if result.httpResp == nil { + result.t.Error("http response is nil") + return + } + if result.httpResp.StatusCode != expected { + result.t.Errorf("expected status code %d, got %d", expected, result.httpResp.StatusCode) + } +} + +func (result *FetchResult) ExpectHeader(key, value string) { + if result.httpResp == nil { + result.t.Error("http response is nil") + return + } + actual := result.httpResp.Header.Get(key) + if actual != value { + result.t.Errorf("expected header %q to be %q, got %q", key, value, actual) + } +} + +func (result *FetchResult) ExpectHelperToReturnHeader(headerName string, validations ...func([]string) error) { + if result.helperResp == nil { + result.t.Error("helper response is nil") + return + } + actual := result.helperResp.Headers[headerName] + for _, validation := range validations { + if err := validation(actual); err != nil { + result.t.Error(err) + } + } +} + +func (result *FetchResult) ExpectBody(expected []byte) { + if !bytes.Equal(result.body, expected) { + result.t.Errorf("expected body to match %v, got %v", string(expected), string(result.body)) + } +} + +func (result *FetchResult) ExpectBodySHA256(expected [sha256.Size]byte) { + if result.httpResp == nil { + result.t.Error("http response is nil") + return + } + hash := sha256.New() + hash.Write(result.body) + actual := hash.Sum(nil) + if !bytes.Equal(actual, expected[:]) { + result.t.Errorf("expected body sha256 to match %x, got %x", expected, actual) + } +} + +func HexToSHA256(t *testing.T, hexStr string) [sha256.Size]byte { + hash, err := hex.DecodeString(hexStr) + if err != nil { + t.Fatalf("failed to decode hex: %v", err) + } + var hash32 [sha256.Size]byte + copy(hash32[:], hash) + return hash32 +} diff --git a/integration/oci_test.go b/integration/oci_test.go new file mode 100644 index 0000000..c7b8d92 --- /dev/null +++ b/integration/oci_test.go @@ -0,0 +1,28 @@ +package integration + +import ( + "context" + "strings" + "testing" +) + +func TestDockerHub(t *testing.T) { + builder := &TestBuilder{} + test := builder.WithDefaultHelperBinary().WithT(t).Build() + ctx := context.Background() + fetchResult, err := test.Fetch(ctx, "https://index.docker.io/v2/library/hello-world/blobs/sha256:d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a") + if err != nil { + t.Fatalf("failed to fetch credentials: %v", err) + } + fetchResult.ExpectHelperToReturnHeader("Authorization", func(s []string) error { + if len(s) != 1 { + t.Errorf("expected exactly one Authorization header, got %d", len(s)) + } + if len(s) == 1 && !strings.HasPrefix(s[0], "Bearer ") { + t.Errorf("expected Authorization header to start with 'Bearer '") + } + return nil + }) + fetchResult.ExpectStatusCode(200) + fetchResult.ExpectBodySHA256(HexToSHA256(t, "d2c94e258dcb3c5ac2798d32e1249e42ef01cba4841c2234249495f87264ac5a")) +} From 6fea6538719564c3536241b1d1f63381c2c9e88e Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:43:50 +0100 Subject: [PATCH 2/4] s3: ignore presinged URLs --- authenticate/s3/s3.go | 5 +++++ helperfactory/fallback/fallback_factory.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/authenticate/s3/s3.go b/authenticate/s3/s3.go index b09c02c..4380cd5 100644 --- a/authenticate/s3/s3.go +++ b/authenticate/s3/s3.go @@ -97,6 +97,11 @@ func (s *S3Resolver) Get(ctx context.Context, req api.GetCredentialsRequest) (ap return api.GetCredentialsResponse{}, err } + if parsedURL.Query().Has("X-Amz-Expires") { + // This is a presigned URL, no need to sign it again. + return api.GetCredentialsResponse{}, nil + } + if parsedURL.Scheme != "https" { return api.GetCredentialsResponse{}, errors.New("only https is supported") } diff --git a/helperfactory/fallback/fallback_factory.go b/helperfactory/fallback/fallback_factory.go index 75d6c46..9510a6a 100644 --- a/helperfactory/fallback/fallback_factory.go +++ b/helperfactory/fallback/fallback_factory.go @@ -31,7 +31,7 @@ func FallbackHelperFactory(rawURL string) (api.Helper, error) { return &authenticateGitHub.GitHub{}, nil case strings.EqualFold(u.Host, "ghcr.io"): return authenticateGitHub.GitHubContainerRegistry(), nil - case strings.HasSuffix(strings.ToLower(u.Host), ".r2.cloudflarestorage.com"): + case strings.HasSuffix(strings.ToLower(u.Host), ".r2.cloudflarestorage.com") && !u.Query().Has("X-Amz-Expires"): return &authenticateS3.R2{}, nil // container registries using the default OCI resolver case strings.EqualFold(u.Host, "index.docker.io"): From 1a06c55f8e6bedc42d4f9b27d94697415c6b4d33 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:56:49 +0100 Subject: [PATCH 3/4] upgrade rules_go --- MODULE.bazel | 3 +-- examples/customized/MODULE.bazel | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 276f58a..9d1d434 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_go", version = "0.50.1") +bazel_dep(name = "rules_go", version = "0.53.0") bazel_dep(name = "gazelle", version = "0.40.0") go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") @@ -16,7 +16,6 @@ use_repo( "com_github_aws_aws_sdk_go_v2", "com_github_aws_aws_sdk_go_v2_config", "com_github_aws_aws_sdk_go_v2_credentials", - "com_github_bazelbuild_rules_go", "com_github_stretchr_testify", "com_github_zalando_go_keyring", "io_k8s_sigs_yaml", diff --git a/examples/customized/MODULE.bazel b/examples/customized/MODULE.bazel index 8592436..2abe047 100644 --- a/examples/customized/MODULE.bazel +++ b/examples/customized/MODULE.bazel @@ -12,7 +12,7 @@ local_path_override( path = "../..", ) -bazel_dep(name = "rules_go", version = "0.50.1") +bazel_dep(name = "rules_go", version = "0.53.0") bazel_dep(name = "gazelle", version = "0.40.0") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") From ddcf7164a162adc15c7272607c0462d52ef6e521 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:59:06 +0100 Subject: [PATCH 4/4] tidy full example --- examples/full/MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/full/MODULE.bazel b/examples/full/MODULE.bazel index 07dfb78..4da2985 100644 --- a/examples/full/MODULE.bazel +++ b/examples/full/MODULE.bazel @@ -68,4 +68,4 @@ oci.pull( image = "hello-world", platforms = ["linux/amd64"], ) -use_repo(oci, "docker_hub_hello_world_linux_amd64", "ghcr_oci") +use_repo(oci, "docker_hub_hello_world", "docker_hub_hello_world_linux_amd64", "ghcr_oci")