s3: respect .tweag-credential-helper.json

authenticate/s3/BUILD.bazel

@@ -7,7 +7,8 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api",
-        "@com_github_aws_aws_sdk_go_v2//aws",
+        "//authenticate/internal/helperconfig",
+        "//authenticate/internal/lookupchain",
         "@com_github_aws_aws_sdk_go_v2//aws/signer/v4:signer",
         "@com_github_aws_aws_sdk_go_v2_config//:config",
         "@com_github_aws_aws_sdk_go_v2_credentials//:credentials",

authenticate/s3/s3.go

@@ -5,76 +5,88 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/tweag/credential-helper/api"
+	"github.com/tweag/credential-helper/authenticate/internal/helperconfig"
+	"github.com/tweag/credential-helper/authenticate/internal/lookupchain"
 )
 
 const (
 	expiresIn   = 15 * time.Minute
 	emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 )
 
-type R2 struct{}
+type S3 struct{}
+
+func (S3) Resolver(ctx context.Context) (api.Resolver, error) {
+	return &S3Resolver{
+		signer: signerv4.NewSigner(),
+	}, nil
+}
 
-func (R2) Resolver(ctx context.Context) (api.Resolver, error) {
-	accessKeyID, ok := os.LookupEnv("R2_ACCESS_KEY_ID")
-	if !ok {
-		return nil, errors.New("R2_ACCESS_KEY_ID not set")
+func (g *S3) SetupInstructionsForURI(ctx context.Context, uri string) string {
+	parsedURL, error := url.Parse(uri)
+	if error != nil {
+		parsedURL = &url.URL{}
 	}
-	// try to use secret access key directly
-	secretAccessKey, ok := os.LookupEnv("R2_SECRET_ACCESS_KEY")
-	if !ok {
-		// try to use cloudflare token
-		cloudflareToken, ok := os.LookupEnv("CLOUDFLARE_API_TOKEN")
-		if !ok {
-			return nil, errors.New("need R2_SECRET_ACCESS_KEY or R2_SECRET_ACCESS_KEY environment variables to access R2")
+
+	var lookupChainInstructions []string
+	cfg, err := configFromContext(ctx, parsedURL)
+	if err == nil {
+		chain := lookupchain.New(cfg.LookupChain)
+		lookupChainInstructions = append(lookupChainInstructions, chain.SetupInstructions(BindigAccessKeyID, "AWS Access Key ID"))
+		lookupChainInstructions = append(lookupChainInstructions, chain.SetupInstructions(BindingSecretAccessKey, "AWS Secret Access Key"))
+		if providerFromHost(parsedURL.Host) == ProviderCloudflareR2 {
+			lookupChainInstructions = append(lookupChainInstructions, chain.SetupInstructions(BindingCloudflareAPIToken, "Cloudflare API Token - can optionally be used to derive the secret access key"))
 		}
-		// cloudflare token can be hashed to obtain the secret access key for the S3 API
-		hasher := sha256.New()
-		hasher.Write([]byte(cloudflareToken))
-		secretAccessKey = hex.EncodeToString(hasher.Sum(nil))
+		lookupChainInstructions = append(lookupChainInstructions, chain.SetupInstructions(BindingRegion, "AWS Region"))
+	} else {
+		lookupChainInstructions = []string{fmt.Sprintf("due to a configuration parsing issue, no further setup instructions are available: %v", err)}
 	}
 
-	cfg, err := config.LoadDefaultConfig(ctx,
-		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")),
-		config.WithRegion("auto"),
-	)
-	if err != nil {
-		return nil, err
+	var serviceString string
+	switch providerFromHost(parsedURL.Host) {
+	case ProviderAWS:
+		serviceString = "AWS S3 object"
+	case ProviderCloudflareR2:
+		serviceString = "Cloudflare R2 object"
+	default:
+		serviceString = "S3-compatible object store object"
 	}
 
-	return &S3Resolver{
-		signer: signerv4.NewSigner(),
-		config: cfg,
-	}, nil
-}
+	return fmt.Sprintf(`%s is a %s.
 
-// CacheKey returns the cache key for the given request.
-// For R2, every object has a unique signature, so the URI is a good cache key.
-func (r *R2) CacheKey(req api.GetCredentialsRequest) string {
-	return req.URI
-}
+The credential helper can be used to download objects from S3 (or S3-compatible object store) buckets.
 
-type S3 struct{}
+IAM Setup:
 
-func (S3) Resolver(ctx context.Context) (api.Resolver, error) {
-	cfg, err := config.LoadDefaultConfig(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return &S3Resolver{
-		signer: signerv4.NewSigner(),
-		config: cfg,
-	}, nil
+  In order to access data from a bucket, you need an AWS user- or service account with read access to the objects you want to access (s3:GetObject).
+  Refer to the AWS documentation for more information: https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-iam.html
+
+Authentication Methods:
+
+  Option 1: Using the AWS CLI and Single Sign On (SSO) as a regular user (Recommended)
+    1. Install the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
+    2. Follow the documentation for using aws configure sso and aws sso login to sign in: https://docs.aws.amazon.com/signin/latest/userguide/command-line-sign-in.html
+    3. Follow the browser prompts to authenticate
+
+  Option 2: Authenticate with other methods:
+    AWS has a lot of ways to authenticate and the credential helper uses the official SDK.
+    If you have more complex requirements, follow the AWS documentation to setup your method of choice:
+
+      https://docs.aws.amazon.com/sdkref/latest/guide/access.html
+
+    This may require you to set environment variables when using Bazel (or other tools).
+
+%s`, uri, serviceString, strings.Join(lookupChainInstructions, "\n\n"))
 }
 
 // CacheKey returns the cache key for the given request.
@@ -85,7 +97,6 @@ func (s *S3) CacheKey(req api.GetCredentialsRequest) string {
 
 type S3Resolver struct {
 	signer signerv4.HTTPSigner
-	config aws.Config
 }
 
 // Get implements the get command of the credential-helper spec:
@@ -106,14 +117,67 @@ func (s *S3Resolver) Get(ctx context.Context, req api.GetCredentialsRequest) (ap
 		return api.GetCredentialsResponse{}, errors.New("only https is supported")
 	}
 
-	region := regionFromHost(parsedURL.Host)
-	if region == "" {
-		return api.GetCredentialsResponse{}, errors.New("unable to determine region from host")
+	cfg, err := configFromContext(ctx, parsedURL)
+	if err != nil {
+		return api.GetCredentialsResponse{}, fmt.Errorf("getting configuration fragment for remotapis helper and url %s: %w", req.URI, err)
+	}
+
+	chain := lookupchain.New(cfg.LookupChain)
+
+	var accessKeyID, secretAccessKey, region string
+
+	if cfg.Region != "" {
+		region = cfg.Region
+	}
+
+	accessKeyIDLookup, err := chain.Lookup(BindigAccessKeyID)
+	if err == nil {
+		accessKeyID = accessKeyIDLookup
+	} else if !lookupchain.IsNotFoundErr(err) {
+		return api.GetCredentialsResponse{}, err
+	}
+
+	if providerFromHost(parsedURL.Host) == ProviderCloudflareR2 {
+		// cloudflare token can be hashed to obtain the secret access key for the S3 API
+		cloudflareAPIToken, err := chain.Lookup(BindingCloudflareAPIToken)
+		if err != nil {
+			return api.GetCredentialsResponse{}, err
+		}
+
+		hasher := sha256.New()
+		hasher.Write([]byte(cloudflareAPIToken))
+		secretAccessKey = hex.EncodeToString(hasher.Sum(nil))
+	}
+
+	secretAccessKeyLookup, err := chain.Lookup(BindingSecretAccessKey)
+	if err == nil {
+		secretAccessKey = secretAccessKeyLookup
+	} else if !lookupchain.IsNotFoundErr(err) {
+		return api.GetCredentialsResponse{}, err
+	}
+
+	regionLookup, err := chain.Lookup(BindingRegion)
+	if err == nil {
+		region = regionLookup
+	} else if !lookupchain.IsNotFoundErr(err) {
+		return api.GetCredentialsResponse{}, err
+	}
+
+	var awsConfigOptions []func(*config.LoadOptions) error
+
+	if len(accessKeyID) > 0 && len(secretAccessKey) > 0 {
+		awsConfigOptions = append(awsConfigOptions,
+			config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")))
+	}
+
+	if len(region) > 0 {
+		awsConfigOptions = append(awsConfigOptions,
+			config.WithRegion(region))
 	}
 
-	s3provider := providerFromHost(parsedURL.Host)
-	if s3provider == ProviderUnknown {
-		return api.GetCredentialsResponse{}, errors.New("unsupported S3 backend")
+	awsConfig, err := config.LoadDefaultConfig(ctx, awsConfigOptions...)
+	if err != nil {
+		return api.GetCredentialsResponse{}, err
 	}
 
 	httpReq := http.Request{
@@ -126,14 +190,14 @@ func (s *S3Resolver) Get(ctx context.Context, req api.GetCredentialsRequest) (ap
 		},
 	}
 
-	cred, err := s.config.Credentials.Retrieve(ctx)
+	cred, err := awsConfig.Credentials.Retrieve(ctx)
 	if err != nil {
 		return api.GetCredentialsResponse{}, err
 	}
 
 	ts := time.Now().UTC()
 
-	if err := s.signer.SignHTTP(ctx, cred, &httpReq, emptySHA256, "s3", region, ts); err != nil {
+	if err := s.signer.SignHTTP(ctx, cred, &httpReq, emptySHA256, "s3", cfg.Region, ts); err != nil {
 		return api.GetCredentialsResponse{}, err
 	}
 
@@ -203,3 +267,95 @@ func providerFromHost(host string) S3Provider {
 
 	return ProviderUnknown
 }
+
+const (
+	BindigAccessKeyID         = "aws-access-key-id"
+	BindingSecretAccessKey    = "aws-secret-access-key"
+	BindingCloudflareAPIToken = "cloudflare-api-token"
+	BindingRegion             = "aws-default-region"
+)
+
+type configFragment struct {
+	// Region is the AWS region to use.
+	// If not set, the region is determined automatically.
+	Region string `json:"region"`
+	// LookupChain defines the order in which secrets are looked up from sources.
+	// Each element is a string that identifies a secret source.
+	// It defaults to the sources "env", "keyring".
+	LookupChain lookupchain.Config `json:"lookup_chain"`
+}
+
+func configFromContext(ctx context.Context, uri *url.URL) (configFragment, error) {
+	sources := []lookupchain.Source{
+		// acces key id
+		&lookupchain.Env{
+			Source:  "env",
+			Name:    "AWS_ACCESS_KEY_ID",
+			Binding: BindigAccessKeyID,
+		},
+		&lookupchain.Keyring{
+			Source:  "env",
+			Service: "tweag-credential-helper:aws-access-key-id",
+			Binding: BindigAccessKeyID,
+		},
+
+		// secret access key
+		&lookupchain.Env{
+			Source:  "env",
+			Name:    "AWS_SECRET_ACCESS_KEY",
+			Binding: BindingSecretAccessKey,
+		},
+		&lookupchain.Keyring{
+			Source:  "env",
+			Service: "tweag-credential-helper:aws-secret-access-key",
+			Binding: BindingSecretAccessKey,
+		},
+
+		// default region
+		&lookupchain.Env{
+			Source:  "env",
+			Name:    "AWS_DEFAULT_REGION",
+			Binding: BindingRegion,
+		},
+		&lookupchain.Keyring{
+			Source:  "env",
+			Service: "tweag-credential-helper:aws-default-region",
+			Binding: BindingRegion,
+		},
+	}
+
+	var cfg configFragment
+
+	switch providerFromHost(uri.Host) {
+	case ProviderAWS:
+		cfg.Region = regionFromHost(uri.Host)
+	case ProviderCloudflareR2:
+		cfg.Region = "auto"
+		sources = append([]lookupchain.Source{
+			&lookupchain.Env{
+				Source:  "env",
+				Name:    "R2_ACCESS_KEY_ID",
+				Binding: BindigAccessKeyID,
+			},
+			&lookupchain.Env{
+				Source:  "env",
+				Name:    "R2_SECRET_ACCESS_KEY",
+				Binding: BindingSecretAccessKey,
+			},
+			&lookupchain.Env{
+				Source:  "env",
+				Name:    "CLOUDFLARE_API_TOKEN",
+				Binding: BindingCloudflareAPIToken,
+			},
+			&lookupchain.Keyring{
+				Source:  "env",
+				Service: "tweag-credential-helper:cloudflare-api-token",
+				Binding: BindingCloudflareAPIToken,
+			},
+		}, sources...)
+	}
+
+	cfg.LookupChain = lookupchain.Default(sources)
+
+	return helperconfig.FromContext(ctx, cfg)
+}

docs/providers/s3.md

@@ -29,6 +29,19 @@ common --credential_helper=s3.amazonaws.com=%workspace%/tools/credential-helper
 common --credential_helper=*.s3.amazonaws.com=%workspace%/tools/credential-helper
 ```
 
+The configuration in `.tweag-credential-helper.json` supports the following values:
+
+- `.urls[].helper`: `"s3"` (name of the helper)
+- `.urls[].region`: The AWS region
+- `.urls[].lookup_chain`: The [lookup chain][lookup_chain] used to find secrets.
+
+In `.tweag-credential-helper.json`, you can use the following secret bindings for the [lookup chain][lookup_chain]:
+
+- `aws-access-key-id`: AWS Access Key ID
+- `aws-secret-access-key`: AWS Secret Access Key
+- `aws-default-region`: AWS region
+- `cloudflare-api-token`: Cloudflare API Token - can optionally be used to derive the secret access key (if secret access key is not provided)
+
 # Troubleshooting
 
 ## HTTP 401 or 403 error codes
@@ -39,4 +52,5 @@ Then ensure your user has access to the object you are trying to access using `a
 [aws-iam]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-iam.html
 [aws-install]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
 [aws-sso-login]: https://docs.aws.amazon.com/signin/latest/userguide/command-line-sign-in.html
-[aws-sdk-auth]: https://docs.aws.amazon.com/sdkref/latest/guide/access.html
+[aws-sdk-auth]: https://docs.aws.amazon.com/sdkref/latest/guide/access.html
+[lookup_chain]: /docs/lookup_chain.md

helperfactory/fallback/fallback_factory.go

@@ -33,7 +33,7 @@ func FallbackHelperFactory(rawURL string) (api.Helper, error) {
 	case strings.EqualFold(u.Host, "ghcr.io"):
 		return authenticateGitHub.GitHubContainerRegistry(), nil
 	case strings.HasSuffix(strings.ToLower(u.Host), ".r2.cloudflarestorage.com") && !u.Query().Has("X-Amz-Expires"):
-		return &authenticateS3.R2{}, nil
+		return &authenticateS3.S3{}, nil
 	case strings.EqualFold(u.Host, "remote.buildbuddy.io"):
 		return &authenticateRemoteAPIs.RemoteAPIs{}, nil
 	// container registries using the default OCI resolver