add support for Cloudflare R2

.github/workflows/ci.yml

@@ -60,6 +60,8 @@ jobs:
       run: bazelisk test //examples:integration_tests
       env:
         GH_TOKEN: ${{ secrets.GH_TOKEN }}
+        R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+        R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
 
   all_ci_tests:
     runs-on: ubuntu-22.04

.gitignore

@@ -3,3 +3,4 @@ agent.sock
 bazel-*
 MODULE.bazel.lock
 .vscode
+.env

MODULE.bazel

@@ -15,6 +15,7 @@ use_repo(
     go_deps,
     "com_github_aws_aws_sdk_go_v2",
     "com_github_aws_aws_sdk_go_v2_config",
+    "com_github_aws_aws_sdk_go_v2_credentials",
     "com_github_stretchr_testify",
     "com_github_zalando_go_keyring",
     "io_k8s_sigs_yaml",

README.md

@@ -45,6 +45,7 @@ sequenceDiagram
 The following providers are supported as of today:
 
 - [AWS S3](/docs/providers/s3.md)
+- [Cloudflare R2](/docs/providers/r2.md)
 - [Google Cloud Storage (GCS)](/docs/providers/gcs.md)
 - [GitHub](/docs/providers/github.md)
 
@@ -79,6 +80,8 @@ common --credential_helper=storage.googleapis.com=%workspace%/tools/credential-h
 # S3
 common --credential_helper=s3.amazonaws.com=%workspace%/tools/credential-helper
 common --credential_helper=*.s3.amazonaws.com=%workspace%/tools/credential-helper
+# Cloudflare R2
+common --credential_helper=*.r2.cloudflarestorage.com=%workspace%/tools/credential-helper
 ```
 
 Simply remove a line if you do not want the credential helper to be used for that service.

authenticate/s3/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "@com_github_aws_aws_sdk_go_v2//aws",
         "@com_github_aws_aws_sdk_go_v2//aws/signer/v4:signer",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
+        "@com_github_aws_aws_sdk_go_v2_credentials//:credentials",
     ],
 )
 

authenticate/s3/s3.go

@@ -2,15 +2,19 @@ package authenticate
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/tweag/credential-helper/api"
 )
 
@@ -19,6 +23,47 @@ const (
 	emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 )
 
+type R2 struct{}
+
+func (R2) Resolver(ctx context.Context) (api.Resolver, error) {
+	accessKeyID, ok := os.LookupEnv("R2_ACCESS_KEY_ID")
+	if !ok {
+		return nil, errors.New("R2_ACCESS_KEY_ID not set")
+	}
+	// try to use secret access key directly
+	secretAccessKey, ok := os.LookupEnv("R2_SECRET_ACCESS_KEY")
+	if !ok {
+		// try to use cloudflare token
+		cloudflareToken, ok := os.LookupEnv("CLOUDFLARE_API_TOKEN")
+		if !ok {
+			return nil, errors.New("need R2_SECRET_ACCESS_KEY or R2_SECRET_ACCESS_KEY environment variables to access R2")
+		}
+		// cloudflare token can be hashed to obtain the secret access key for the S3 API
+		hasher := sha256.New()
+		hasher.Write([]byte(cloudflareToken))
+		secretAccessKey = hex.EncodeToString(hasher.Sum(nil))
+	}
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")),
+		config.WithRegion("auto"),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &S3Resolver{
+		signer: signerv4.NewSigner(),
+		config: cfg,
+	}, nil
+}
+
+// CacheKey returns the cache key for the given request.
+// For R2, every object has a unique signature, so the URI is a good cache key.
+func (r *R2) CacheKey(req api.GetCredentialsRequest) string {
+	return req.URI
+}
+
 type S3 struct{}
 
 func (S3) Resolver(ctx context.Context) (api.Resolver, error) {
@@ -61,6 +106,11 @@ func (s *S3Resolver) Get(ctx context.Context, req api.GetCredentialsRequest) (ap
 		return api.GetCredentialsResponse{}, errors.New("unable to determine region from host")
 	}
 
+	s3provider := providerFromHost(parsedURL.Host)
+	if s3provider == ProviderUnknown {
+		return api.GetCredentialsResponse{}, errors.New("unsupported S3 backend")
+	}
+
 	httpReq := http.Request{
 		Method: http.MethodGet,
 		URL:    parsedURL,
@@ -89,6 +139,12 @@ func (s *S3Resolver) Get(ctx context.Context, req api.GetCredentialsRequest) (ap
 }
 
 func regionFromHost(host string) string {
+	// cloudfare r2
+	if strings.HasSuffix(host, ".r2.cloudflarestorage.com") {
+		return "auto"
+	}
+
+	// AWS S3
 	if host == "s3.amazonaws.com" {
 		return "us-east-1"
 	}
@@ -122,3 +178,23 @@ func regionFromHost(host string) string {
 
 	return ""
 }
+
+type S3Provider int
+
+const (
+	ProviderUnknown S3Provider = iota
+	ProviderAWS
+	ProviderCloudflareR2
+)
+
+func providerFromHost(host string) S3Provider {
+	if strings.HasSuffix(host, ".r2.cloudflarestorage.com") {
+		return ProviderCloudflareR2
+	}
+
+	if strings.HasSuffix(host, ".amazonaws.com") {
+		return ProviderAWS
+	}
+
+	return ProviderUnknown
+}

docs/providers/r2.md

@@ -0,0 +1,21 @@
+# Cloudflare R2
+
+This document explains how to setup your system for authenticating to R2 using the credential helper.
+
+## Authentication
+
+R2 has an S3-compatible API that requires the creation of special tokens. These are different from normal user tokens and are only used for R2.ß
+
+- Create an API token by [following the documentation][r2-auth-tokens]
+- Set the `R2_ACCESS_KEY_ID` and `R2_SECRET_ACCESS_KEY` environment variables when running Bazel or other tools
+
+## Configuration
+
+Add to your `.bazelrc`:
+
+```
+common --credential_helper=*.r2.cloudflarestorage.com=%workspace%/tools/credential-helper
+```
+
+[wrangler-install]: https://developers.cloudflare.com/workers/wrangler/install-and-update/
+[r2-auth-tokens]: https://developers.cloudflare.com/r2/api/s3/tokens/

docs/providers/s3.md

@@ -1,6 +1,6 @@
 # AWS Simple Storage Service (S3)
 
-This document explains how to setup you system for authenticating to S3 using the credential helper.
+This document explains how to setup your system for authenticating to S3 using the credential helper.
 
 ## IAM Setup
 

examples/BUILD.bazel

@@ -33,15 +33,17 @@ default_test_runner(
 bazel_integration_tests(
     name = "full_test",
     additional_env_inherit = [
+        "AWS_ACCESS_KEY_ID",
         "AWS_DEFAULT_REGION",
         "AWS_REGION",
-        "AWS_ACCESS_KEY_ID",
         "AWS_SECRET_ACCESS_KEY",
         "AWS_SESSION_TOKEN",
         "CREDENTIAL_HELPER_LOGGING",
         "GH_TOKEN",
         "GITHUB_TOKEN",
         "GOOGLE_APPLICATION_CREDENTIALS",
+        "R2_ACCESS_KEY_ID",
+        "R2_SECRET_ACCESS_KEY",
     ],
     bazel_versions = bazel_binaries.versions.all,
     test_runner = ":default_test_runner",

examples/full/.bazelrc

@@ -3,4 +3,5 @@ common --credential_helper=github.com=%workspace%/tools/credential-helper
 common --credential_helper=raw.githubusercontent.com=%workspace%/tools/credential-helper
 common --credential_helper=s3.amazonaws.com=%workspace%/tools/credential-helper
 common --credential_helper=*.s3.amazonaws.com=%workspace%/tools/credential-helper
+common --credential_helper=*.r2.cloudflarestorage.com=%workspace%/tools/credential-helper
 common --test_output=errors

examples/full/BUILD.bazel

@@ -18,6 +18,16 @@ sh_test(
     data = ["@hello_world_s3//file"],
 )
 
+sh_test(
+    name = "check_r2",
+    srcs = ["@tweag-credential-helper//examples/testing:check_file_hash.sh"],
+    args = [
+        "$(location @hello_world_r2//file)",
+        "df4bf34acd7328fa6f0b6913dcc56b450b6888a562718c8bf970e9b9685114f5",
+    ],
+    data = ["@hello_world_r2//file"],
+)
+
 sh_test(
     name = "check_gcs",
     srcs = ["@tweag-credential-helper//examples/testing:check_file_hash.sh"],

examples/full/MODULE.bazel

@@ -26,6 +26,12 @@ http_file(
     urls = ["https://malte-s3-bazel-test.s3.amazonaws.com/hello_world"],
 )
 
+http_file(
+    name = "hello_world_r2",
+    integrity = "sha256-30vzSs1zKPpvC2kT3MVrRQtoiKVicYyL+XDpuWhRFPU=",
+    urls = ["https://47ffb5f0a156e14a0985028c836501e2.r2.cloudflarestorage.com/r2-private-example/hello_world"],
+)
+
 http_file(
     name = "hello_world_gcs",
     integrity = "sha256-/Llq5Bi4Y+uTiDqUzDn4FUXP35fiDSku0b2PeKFTztw=",

go.mod

@@ -5,7 +5,7 @@ go 1.22.5
 require (
 	github.com/aws/aws-sdk-go-v2 v1.32.5
 	github.com/aws/aws-sdk-go-v2/config v1.28.5
-	github.com/bazelbuild/rules_go v0.50.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.46
 	github.com/stretchr/testify v1.10.0
 	github.com/zalando/go-keyring v0.2.6
 	golang.org/x/oauth2 v0.24.0
@@ -15,7 +15,6 @@ require (
 require (
 	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect

go.sum

@@ -28,8 +28,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLb
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg=
 github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
 github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/bazelbuild/rules_go v0.50.1 h1:/BUvuaB8MEiUA2oLPPCGtuw5V+doAYyiGTFyoSWlkrw=
-github.com/bazelbuild/rules_go v0.50.1/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0=
 github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

helperfactory/fallback/fallback_factory.go

@@ -28,6 +28,8 @@ func FallbackHelperFactory(rawURL string) (api.Helper, error) {
 		fallthrough
 	case strings.EqualFold(u.Host, "raw.githubusercontent.com"):
 		return &authenticateGitHub.GitHub{}, nil
+	case strings.HasSuffix(strings.ToLower(u.Host), ".r2.cloudflarestorage.com"):
+		return &authenticateS3.R2{}, nil
 	default:
 		logging.Basicf("no matching credential helper found for %s - returning empty response\n", rawURL)
 		return authenticateNull.Null{}, nil

testdata/r2.json

@@ -0,0 +1,3 @@
+{
+    "uri": "https://47ffb5f0a156e14a0985028c836501e2.r2.cloudflarestorage.com/r2-private-example/hello_world"
+}