From 48958b2c07bea72bc5fa7f00f64d083fe2add33f Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Sat, 22 Feb 2025 17:22:02 +0100 Subject: [PATCH 1/8] add testdata for BuildBuddy --- testdata/buildbuddy.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 testdata/buildbuddy.json diff --git a/testdata/buildbuddy.json b/testdata/buildbuddy.json new file mode 100644 index 0000000..821e872 --- /dev/null +++ b/testdata/buildbuddy.json @@ -0,0 +1 @@ +{"uri": "https://remote.buildbuddy.io/build.bazel.remote.execution.v2.Execution"} From 7fe266f7a657f6707f90bd45e8ef3582dac6f1a8 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:23:22 +0100 Subject: [PATCH 2/8] add support for remote execution APIs --- authenticate/remoteapis/BUILD.bazel | 20 +++ authenticate/remoteapis/remoteapis.go | 154 ++++++++++++++++++ bzl/private/source_files/BUILD.bazel | 1 + cmd/root/root.go | 1 + config/BUILD.bazel | 1 + config/config.go | 2 + helperfactory/string/BUILD.bazel | 1 + helperfactory/string/helperstring.go | 3 + .../remote_execution_v2_capabilities.json | 1 + 9 files changed, 184 insertions(+) create mode 100644 authenticate/remoteapis/BUILD.bazel create mode 100644 authenticate/remoteapis/remoteapis.go create mode 100644 testdata/remote_execution_v2_capabilities.json diff --git a/authenticate/remoteapis/BUILD.bazel b/authenticate/remoteapis/BUILD.bazel new file mode 100644 index 0000000..372d837 --- /dev/null +++ b/authenticate/remoteapis/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "remoteapis", + srcs = ["remoteapis.go"], + importpath = "github.com/tweag/credential-helper/authenticate/remoteapis", + visibility = ["//visibility:public"], + deps = [ + "//api", + "//authenticate/internal/helperconfig", + "//authenticate/internal/lookupchain", + "//logging", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//:__subpackages__"], +) diff --git a/authenticate/remoteapis/remoteapis.go b/authenticate/remoteapis/remoteapis.go new file mode 100644 index 0000000..0cf5e20 --- /dev/null +++ b/authenticate/remoteapis/remoteapis.go @@ -0,0 +1,154 @@ +package remoteapis + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/tweag/credential-helper/api" + "github.com/tweag/credential-helper/authenticate/internal/helperconfig" + "github.com/tweag/credential-helper/authenticate/internal/lookupchain" + "github.com/tweag/credential-helper/logging" +) + +// well-known grpc names (name of the Java package and the name of the service in the .proto file) +const ( + GOOGLE_BYTESTREAM_BYTESTREAM = "google.bytestream.ByteStream" + GOOGLE_DEVTOOLS_BUILD_V1_PUBLISHBUILDEVENT = "google.devtools.build.v1.PublishBuildEvent" + REMOTE_ASSET_V1_FETCH = "build.bazel.remote.asset.v1.Fetch" + REMOTE_ASSET_V1_PUSH = "build.bazel.remote.asset.v1.Push" + REMOTE_EXECUTION_V2_ACTIONCACHE = "build.bazel.remote.execution.v2.ActionCache" + REMOTE_EXECUTION_V2_CAPABILITIES = "build.bazel.remote.execution.v2.Capabilities" + REMOTE_EXECUTION_V2_CONTENTADDRESSABLESTORAGE = "build.bazel.remote.execution.v2.ContentAddressableStorage" + REMOTE_EXECUTION_V2_EXECUTION = "build.bazel.remote.execution.v2.Execution" +) + +type RemoteAPIs struct{} + +// CacheKey returns a cache key for the given request. +// For remote apis, the full URI is a good cache key. +func (g *RemoteAPIs) CacheKey(req api.GetCredentialsRequest) string { + return req.URI +} + +func (RemoteAPIs) Resolver(ctx context.Context) (api.Resolver, error) { + return &RemoteAPIs{}, nil +} + +// Get implements the get command of the credential-helper spec: +// +// https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md#get +func (g *RemoteAPIs) Get(ctx context.Context, req api.GetCredentialsRequest) (api.GetCredentialsResponse, error) { + cfg, err := helperconfig.FromContext(ctx, configFragment{ + AuthMethod: "header", + LookupChain: lookupchain.Default([]lookupchain.Source{ + &lookupchain.Env{ + Source: "env", + Name: "CREDENTIAL_HELPER_REMOTEAPIS_SECRET", + Binding: "default", + }, + &lookupchain.Keyring{ + Source: "keyring", + Service: "tweag-credential-helper:remoteapis", + Binding: "default", + }, + }), + }) + if err != nil { + return api.GetCredentialsResponse{}, fmt.Errorf("getting configuration fragment for remotapis helper and url %s: %w", req.URI, err) + } + chain := lookupchain.New(cfg.LookupChain) + secret, err := chain.Lookup("default") + if err != nil { + return api.GetCredentialsResponse{}, err + } + + parsedURL, error := url.Parse(req.URI) + if error != nil { + return api.GetCredentialsResponse{}, error + } + + // the scheme for remote APIs appears to be https for all of the following: + // - https:// + // - grpc:// + // - grpcs:// + // + // only unencrypted http:// (using the HTTP/1.1 cache protocl) uses a different scheme + // for simplicity, we only support the grpc(s) remote APIs here + if parsedURL.Scheme != "https" && parsedURL.Scheme != "grpc" && parsedURL.Scheme != "grpcs" { + if parsedURL.Scheme == "grpc" || parsedURL.Scheme == "grpcs" { + logging.Errorf("expecting to see https scheme (which is the URI that Bazel norrmally forwards to the credential helper for remoteapis), but got %q", parsedURL.Scheme) + } else { + return api.GetCredentialsResponse{}, fmt.Errorf("only https, grpc, and grpcs are supported, but got %q", parsedURL.Scheme) + } + } + + // the following only works for grpc (and not HTTP/1.1) + rpcName, hasPrefix := strings.CutPrefix(parsedURL.Path, "/") + if !hasPrefix { + return api.GetCredentialsResponse{}, errors.New("remote execution API path must start with /") + } + switch rpcName { + default: + return api.GetCredentialsResponse{}, fmt.Errorf("unknown remote execution API path %q - maybe you are trying to use HTTP/1.1, but currently only gRPC is supported", parsedURL.Path) + case GOOGLE_BYTESTREAM_BYTESTREAM: + case GOOGLE_DEVTOOLS_BUILD_V1_PUBLISHBUILDEVENT: + case REMOTE_ASSET_V1_FETCH: + case REMOTE_ASSET_V1_PUSH: + case REMOTE_EXECUTION_V2_ACTIONCACHE: + case REMOTE_EXECUTION_V2_CAPABILITIES: + case REMOTE_EXECUTION_V2_CONTENTADDRESSABLESTORAGE: + case REMOTE_EXECUTION_V2_EXECUTION: + } + + headerName := cfg.HeaderName + secretEncoding := func(secret string) string { + // by default, the secret is directly used as a header value + return secret + } + switch cfg.AuthMethod { + case "header": + if len(cfg.HeaderName) == 0 { + return api.GetCredentialsResponse{}, errors.New(`header_name must be set for auth method "header"`) + } + case "basic_auth": + // bazel-remote only supports basic auth. + // It tries to read the standard grpc metadata key ":authority" to get the username and password. + // This is special header that the credential helper cannot provide. + // As a fallback for proxies, bazel-remote also reads the grpc metadata key "authorization" to get the username and password encoded as a base64 string. + if len(cfg.HeaderName) > 0 { + headerName = cfg.HeaderName + } else { + headerName = "authorization" + } + secretEncoding = func(secret string) string { + return "Basic " + base64.StdEncoding.EncodeToString([]byte(secret)) + } + default: + return api.GetCredentialsResponse{}, fmt.Errorf(`unknown auth method %q. Possible values are "header" and "basic_auth"`, cfg.AuthMethod) + } + + return api.GetCredentialsResponse{ + Headers: map[string][]string{ + headerName: {secretEncoding(secret)}, + }, + }, nil +} + +type configFragment struct { + // AuthMethod is the method used to authenticate with the remote API. + // Valid values are: + // - "header", which works BuildBuddy and other services that use a HTTP header directly. The secret is used as the value of the header. + // - "basic_auth", which works for bazel-remote. The secret should be of the form "username:password". + // It defaults to "header". + AuthMethod string `json:"auth_method"` + // HeaderName is the name of the header to set the secret in. + HeaderName string `json:"header_name"` + // LookupChain defines the order in which secrets are looked up from sources. + // Each element is a string that identifies a secret source. + // It defaults to the sources "env", "keyring". + LookupChain lookupchain.Config `json:"lookup_chain"` +} diff --git a/bzl/private/source_files/BUILD.bazel b/bzl/private/source_files/BUILD.bazel index 967af96..6d8a960 100644 --- a/bzl/private/source_files/BUILD.bazel +++ b/bzl/private/source_files/BUILD.bazel @@ -12,6 +12,7 @@ release_files = [ "//authenticate/internal/lookupchain:all_files", "//authenticate/null:all_files", "//authenticate/oci:all_files", + "//authenticate/remoteapis:all_files", "//authenticate/s3:all_files", "//bzl:all_files", "//bzl/config:all_files", diff --git a/cmd/root/root.go b/cmd/root/root.go index 36b6501..95bb562 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -72,6 +72,7 @@ func foreground(ctx context.Context, helperFactory api.HelperFactory, cache api. cfg, err := configReader.Read() if err == nil { + logging.Debugf("found config file and choosing helper from it") helperFactory = func(uri string) (api.Helper, error) { helper, helperConfig, err := cfg.FindHelper(uri) if err != nil { diff --git a/config/BUILD.bazel b/config/BUILD.bazel index 93986e1..b20d84e 100644 --- a/config/BUILD.bazel +++ b/config/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "//agent/locate", "//api", "//helperfactory/string", + "//logging", ], ) diff --git a/config/config.go b/config/config.go index ec7fe93..66f2eb3 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ import ( "github.com/tweag/credential-helper/agent/locate" "github.com/tweag/credential-helper/api" helperstringfactory "github.com/tweag/credential-helper/helperfactory/string" + "github.com/tweag/credential-helper/logging" ) var ErrConfigNotFound = errors.New("config file not found") @@ -54,6 +55,7 @@ func (c Config) FindHelper(uri string) (api.Helper, []byte, error) { } helper := helperstringfactory.HelperFromString(urlConfig.Helper) if helper != nil { + logging.Debugf("selected helper %s from config", urlConfig.Helper) return helper, urlConfig.Config, nil } return nil, nil, fmt.Errorf("unknown helper: %s", urlConfig.Helper) diff --git a/helperfactory/string/BUILD.bazel b/helperfactory/string/BUILD.bazel index 8f4453b..9bc2608 100644 --- a/helperfactory/string/BUILD.bazel +++ b/helperfactory/string/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//authenticate/github", "//authenticate/null", "//authenticate/oci", + "//authenticate/remoteapis", "//authenticate/s3", ], ) diff --git a/helperfactory/string/helperstring.go b/helperfactory/string/helperstring.go index c211870..99c86c6 100644 --- a/helperfactory/string/helperstring.go +++ b/helperfactory/string/helperstring.go @@ -7,6 +7,7 @@ import ( authenticateGitHub "github.com/tweag/credential-helper/authenticate/github" authenticateNull "github.com/tweag/credential-helper/authenticate/null" authenticateOCI "github.com/tweag/credential-helper/authenticate/oci" + authenticateRemoteAPIs "github.com/tweag/credential-helper/authenticate/remoteapis" authenticateS3 "github.com/tweag/credential-helper/authenticate/s3" ) @@ -20,6 +21,8 @@ func HelperFromString(s string) api.Helper { return &authenticateGitHub.GitHub{} case "oci": return authenticateOCI.NewFallbackOCI() + case "remoteapis": + return &authenticateRemoteAPIs.RemoteAPIs{} case "null": return &authenticateNull.Null{} } diff --git a/testdata/remote_execution_v2_capabilities.json b/testdata/remote_execution_v2_capabilities.json new file mode 100644 index 0000000..2d46a52 --- /dev/null +++ b/testdata/remote_execution_v2_capabilities.json @@ -0,0 +1 @@ +{"uri": "https://192.168.178.43:9092/build.bazel.remote.execution.v2.Capabilities"} From ce71a1f7c4be2979255351c571468242c89fc0ab Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Sat, 22 Feb 2025 15:18:19 +0100 Subject: [PATCH 3/8] implement singleton registry that knows about all helpers --- bzl/private/source_files/BUILD.bazel | 2 +- config/BUILD.bazel | 2 +- config/config.go | 6 +-- docs/plugins.md | 11 ++++ .../helper/authenticate/BUILD.bazel | 5 +- .../helper/authenticate/pathtoheader.go | 8 +++ helperfactory/string/helperstring.go | 30 ----------- .../string => registry}/BUILD.bazel | 6 +-- registry/registry.go | 53 +++++++++++++++++++ 9 files changed, 84 insertions(+), 39 deletions(-) delete mode 100644 helperfactory/string/helperstring.go rename {helperfactory/string => registry}/BUILD.bazel (76%) create mode 100644 registry/registry.go diff --git a/bzl/private/source_files/BUILD.bazel b/bzl/private/source_files/BUILD.bazel index 6d8a960..b422ed6 100644 --- a/bzl/private/source_files/BUILD.bazel +++ b/bzl/private/source_files/BUILD.bazel @@ -28,9 +28,9 @@ release_files = [ "//config:all_files", "//helperfactory:all_files", "//helperfactory/fallback:all_files", - "//helperfactory/string:all_files", "//installer:all_files", "//logging:all_files", + "//registry:all_files", ] dev_files = [ diff --git a/config/BUILD.bazel b/config/BUILD.bazel index b20d84e..6213e06 100644 --- a/config/BUILD.bazel +++ b/config/BUILD.bazel @@ -8,8 +8,8 @@ go_library( deps = [ "//agent/locate", "//api", - "//helperfactory/string", "//logging", + "//registry", ], ) diff --git a/config/config.go b/config/config.go index 66f2eb3..cdbc830 100644 --- a/config/config.go +++ b/config/config.go @@ -10,8 +10,8 @@ import ( "github.com/tweag/credential-helper/agent/locate" "github.com/tweag/credential-helper/api" - helperstringfactory "github.com/tweag/credential-helper/helperfactory/string" "github.com/tweag/credential-helper/logging" + "github.com/tweag/credential-helper/registry" ) var ErrConfigNotFound = errors.New("config file not found") @@ -53,7 +53,7 @@ func (c Config) FindHelper(uri string) (api.Helper, []byte, error) { if len(urlConfig.Path) > 0 && !globMatch(urlConfig.Path, requested.Path) { continue } - helper := helperstringfactory.HelperFromString(urlConfig.Helper) + helper := registry.HelperFromString(urlConfig.Helper) if helper != nil { logging.Debugf("selected helper %s from config", urlConfig.Helper) return helper, urlConfig.Config, nil @@ -62,7 +62,7 @@ func (c Config) FindHelper(uri string) (api.Helper, []byte, error) { } // this is equivalent to null.Null{} // but avoids the import of the null package - return helperstringfactory.HelperFromString("null"), nil, nil + return registry.HelperFromString("null"), nil, nil } type ConfigReader interface { diff --git a/docs/plugins.md b/docs/plugins.md index f9160aa..a9e2a7d 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -27,6 +27,17 @@ You can find the built-in default implementations under [/authenticate][authenti The credential helper supports multiple authentication providers in a single helper binary. The factory function determines which `api.Helper` to use based on the request URI. You can find the default implementation in [github.com/tweag/credential-helper/helperfactory/fallback.FallbackHelperFactory][fallback-helper-factory]. +## Registering your helper + +Custom helpers need to be registered at program startup to work and be recognized correctly. For this, the `registry.Register` function can be called to add a globally known register (with a unique name) to the registry. +Simply add an `init` function to the package that implements `api.Helper`: + +```go +func init() { + registry.Register("foo", FooHelper{}) +} +``` + ## (Optional) Replace the default in-memory cache By default, the agent process uses a simple in-memory key-value store to cache credentials. You can provide a custom implementation to persist credentials (on disk, in a database, in a shared key-value store, using a (hardware backed) secure storage, etc.), implement more selective caching (decide what to keep), or perform other custom logic. diff --git a/examples/customized/helper/authenticate/BUILD.bazel b/examples/customized/helper/authenticate/BUILD.bazel index 31c9122..a51d31f 100644 --- a/examples/customized/helper/authenticate/BUILD.bazel +++ b/examples/customized/helper/authenticate/BUILD.bazel @@ -5,5 +5,8 @@ go_library( srcs = ["pathtoheader.go"], importpath = "github.com/tweag/credential-helper/examples/customized/helper/authenticate", visibility = ["//visibility:public"], - deps = ["@tweag-credential-helper//api"], + deps = [ + "@tweag-credential-helper//api", + "@tweag-credential-helper//registry", + ], ) diff --git a/examples/customized/helper/authenticate/pathtoheader.go b/examples/customized/helper/authenticate/pathtoheader.go index 28c6e81..e32c518 100644 --- a/examples/customized/helper/authenticate/pathtoheader.go +++ b/examples/customized/helper/authenticate/pathtoheader.go @@ -9,8 +9,16 @@ import ( "time" "github.com/tweag/credential-helper/api" + "github.com/tweag/credential-helper/registry" ) +func init() { + // This code runs when the program starts. + // It registers this helper with the registry under the name `pathtoheader`. + // The registry is used by the agent to look up helpers by name. + registry.Register("pathtoheader", PathToHeader{}) +} + // PathToHeader is a credential helper that // takes a request path and returns // a custom header including it. diff --git a/helperfactory/string/helperstring.go b/helperfactory/string/helperstring.go deleted file mode 100644 index 99c86c6..0000000 --- a/helperfactory/string/helperstring.go +++ /dev/null @@ -1,30 +0,0 @@ -package string - -import ( - "github.com/tweag/credential-helper/api" - - authenticateGCS "github.com/tweag/credential-helper/authenticate/gcs" - authenticateGitHub "github.com/tweag/credential-helper/authenticate/github" - authenticateNull "github.com/tweag/credential-helper/authenticate/null" - authenticateOCI "github.com/tweag/credential-helper/authenticate/oci" - authenticateRemoteAPIs "github.com/tweag/credential-helper/authenticate/remoteapis" - authenticateS3 "github.com/tweag/credential-helper/authenticate/s3" -) - -func HelperFromString(s string) api.Helper { - switch s { - case "s3": - return &authenticateS3.S3{} - case "gcs": - return &authenticateGCS.GCS{} - case "github": - return &authenticateGitHub.GitHub{} - case "oci": - return authenticateOCI.NewFallbackOCI() - case "remoteapis": - return &authenticateRemoteAPIs.RemoteAPIs{} - case "null": - return &authenticateNull.Null{} - } - return nil -} diff --git a/helperfactory/string/BUILD.bazel b/registry/BUILD.bazel similarity index 76% rename from helperfactory/string/BUILD.bazel rename to registry/BUILD.bazel index 9bc2608..94d902c 100644 --- a/helperfactory/string/BUILD.bazel +++ b/registry/BUILD.bazel @@ -1,9 +1,9 @@ load("@rules_go//go:def.bzl", "go_library") go_library( - name = "string", - srcs = ["helperstring.go"], - importpath = "github.com/tweag/credential-helper/helperfactory/string", + name = "registry", + srcs = ["registry.go"], + importpath = "github.com/tweag/credential-helper/registry", visibility = ["//visibility:public"], deps = [ "//api", diff --git a/registry/registry.go b/registry/registry.go new file mode 100644 index 0000000..3a91dad --- /dev/null +++ b/registry/registry.go @@ -0,0 +1,53 @@ +package registry + +import ( + "github.com/tweag/credential-helper/api" + authenticateGCS "github.com/tweag/credential-helper/authenticate/gcs" + authenticateGitHub "github.com/tweag/credential-helper/authenticate/github" + authenticateNull "github.com/tweag/credential-helper/authenticate/null" + authenticateOCI "github.com/tweag/credential-helper/authenticate/oci" + authenticateRemoteAPIs "github.com/tweag/credential-helper/authenticate/remoteapis" + authenticateS3 "github.com/tweag/credential-helper/authenticate/s3" +) + +var singleton = Helpers{ + Map: map[string]api.Helper{ + "gcs": &authenticateGCS.GCS{}, + "github": &authenticateGitHub.GitHub{}, + "null": &authenticateNull.Null{}, + "oci": authenticateOCI.NewFallbackOCI(), + "remoteapis": &authenticateRemoteAPIs.RemoteAPIs{}, + "s3": &authenticateS3.S3{}, + }, +} + +// HelperFromString returns the helper corresponding to the given string. +func HelperFromString(s string) api.Helper { + return singleton.Map[s] +} + +// Register registers a new helper with the given name. +func Register(name string, helper api.Helper) { + singleton.register(name, helper) +} + +// Names returns the names of all registered helpers. +func Names() []string { + return singleton.names() +} + +type Helpers struct { + Map map[string]api.Helper +} + +func (h *Helpers) register(name string, helper api.Helper) { + h.Map[name] = helper +} + +func (h *Helpers) names() []string { + names := make([]string, 0, len(h.Map)) + for name := range h.Map { + names = append(names, name) + } + return names +} From 0c3d595b409dc6bbf301428ae7b527804f44b414 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:41:51 +0100 Subject: [PATCH 4/8] add "setup-keyring" subcommand This command can be used to store secrets in the system keyring. Examples: $ echo -ne "user:pass" | credential-helper setup-keyring tweag-credential-helper:remoteapis $ credential-helper setup-keyring --file github_secret.txt gh:github.com --- agent/locate/locate.go | 16 ++++ api/api.go | 8 ++ bzl/private/source_files/BUILD.bazel | 1 + cmd/root/BUILD.bazel | 1 + cmd/root/root.go | 10 ++- cmd/setup/BUILD.bazel | 18 +++++ cmd/setup/keyring.go | 110 +++++++++++++++++++++++++++ 7 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 cmd/setup/BUILD.bazel create mode 100644 cmd/setup/keyring.go diff --git a/agent/locate/locate.go b/agent/locate/locate.go index 8b4a620..9ef1d9b 100644 --- a/agent/locate/locate.go +++ b/agent/locate/locate.go @@ -17,6 +17,14 @@ import ( // environment variables to ensure // a consistent working environment. func SetupEnvironment() error { + originalWorkingDirectory, err := os.Getwd() + if err != nil { + return err + } + if err := os.Setenv(api.OriginalWorkingDirectoryEnv, originalWorkingDirectory); err != nil { + return err + } + workspacePath, err := setupWorkspaceDirectory() if err != nil { return err @@ -111,6 +119,14 @@ func AgentPaths() (string, string) { return socketPath, pidPath } +func RemapToOriginalWorkingDirectory(p string) string { + if filepath.IsAbs(p) { + return p + } + prefix := os.Getenv(api.OriginalWorkingDirectoryEnv) + return filepath.Join(prefix, p) +} + func tmpDir() string { // In Bazel integration tests we can't (and shouldn't) // touch /tmp diff --git a/api/api.go b/api/api.go index 451e1c7..2396a21 100644 --- a/api/api.go +++ b/api/api.go @@ -72,6 +72,11 @@ type Cache interface { Prune(context.Context) error } +// URISetupper is an optional interface that can be implemented by helpers to perform setup for a given URI. +type URISetupper interface { + SetupInstructionsForURI(uri string) string +} + var CacheMiss = errors.New("cache miss") // Environment variable names used by the credential helper. @@ -90,6 +95,9 @@ const ( WorkdirEnv = "CREDENTIAL_HELPER_WORKDIR" // The working directory of Bazel (path containing root module). WorkspaceEnv = "CREDENTIAL_HELPER_WORKSPACE_DIRECTORY" + // The cwd of the helper process before it is chdir'd into the workdir. + // This is used to resolve relative paths for some CLI commands. + OriginalWorkingDirectoryEnv = "CREDENTIAL_HELPER_ORIGINAL_WORKING_DIRECTORY" ) // Placeholders in configuration that is expanded automatically. diff --git a/bzl/private/source_files/BUILD.bazel b/bzl/private/source_files/BUILD.bazel index b422ed6..7b35238 100644 --- a/bzl/private/source_files/BUILD.bazel +++ b/bzl/private/source_files/BUILD.bazel @@ -25,6 +25,7 @@ release_files = [ "//cmd/credential-helper:all_files", "//cmd/installer:all_files", "//cmd/root:all_files", + "//cmd/setup:all_files", "//config:all_files", "//helperfactory:all_files", "//helperfactory/fallback:all_files", diff --git a/cmd/root/BUILD.bazel b/cmd/root/BUILD.bazel index eb768b1..433e3c8 100644 --- a/cmd/root/BUILD.bazel +++ b/cmd/root/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//api", "//cache", "//cmd/installer", + "//cmd/setup", "//config", "//logging", ], diff --git a/cmd/root/root.go b/cmd/root/root.go index 95bb562..70b6414 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -16,12 +16,16 @@ import ( "github.com/tweag/credential-helper/api" "github.com/tweag/credential-helper/cache" "github.com/tweag/credential-helper/cmd/installer" + "github.com/tweag/credential-helper/cmd/setup" "github.com/tweag/credential-helper/config" "github.com/tweag/credential-helper/logging" ) -const usage = `Usage: - credential-helper get` +const usage = `Usage: credential-helper [COMMAND] [ARGS...] + +Commands: + get get credentials in the form of http headers for the uri provided on stdin and print result to stdout (see https://github.com/EngFlow/credential-helper-spec for more information) + setup-keyring stores a secret in the system keyring` func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewCache, args []string) { setLogLevel() @@ -40,6 +44,8 @@ func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewC switch command { case "get": clientProcess(ctx, helperFactory) + case "setup-keyring": + setup.KeyringProcess(args[2:]) case "agent-launch": agentProcess(ctx, newCache) case "agent-shutdown": diff --git a/cmd/setup/BUILD.bazel b/cmd/setup/BUILD.bazel new file mode 100644 index 0000000..44366ce --- /dev/null +++ b/cmd/setup/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "setup", + srcs = ["keyring.go"], + importpath = "github.com/tweag/credential-helper/cmd/setup", + visibility = ["//visibility:public"], + deps = [ + "//agent/locate", + "@com_github_zalando_go_keyring//:go-keyring", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//:__subpackages__"], +) diff --git a/cmd/setup/keyring.go b/cmd/setup/keyring.go new file mode 100644 index 0000000..bef73ef --- /dev/null +++ b/cmd/setup/keyring.go @@ -0,0 +1,110 @@ +package setup + +import ( + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/tweag/credential-helper/agent/locate" + keyring "github.com/zalando/go-keyring" +) + +// KeyringProcess is the entry point for the setup-keyring command. +// It takes a string key (name of the secret) and a string value (the secret itself) and stores it in the keyring. +func KeyringProcess(args []string) { + var sourceFilePath string + var read bool + var clear bool + + flagSet := flag.NewFlagSet("setup-keyring", flag.ExitOnError) + flagSet.Usage = func() { + fmt.Fprintf(flagSet.Output(), "Stores a secret read from a file or stdin in the system keyring under the name specified by service.\n\n") + fmt.Fprintf(flagSet.Output(), "Usage: credential-helper setup-keyring [--file file] [--read | --clear] [service]\n") + flagSet.PrintDefaults() + examples := []string{ + "credential-helper setup-keyring gh:github.com < secret.txt", + "credential-helper setup-keyring --clear gh:github.com", + "credential-helper setup-keyring --file secret.txt tweag-credential-helper:remoteapis", + "credential-helper setup-keyring --read tweag-credential-helper:buildbuddy_api_key", + } + fmt.Fprintf(flagSet.Output(), "\nExamples:\n") + for _, example := range examples { + fmt.Fprintf(flagSet.Output(), " $ %s\n", example) + } + os.Exit(1) + } + flagSet.StringVar(&sourceFilePath, "file", "", "File to read the secret from") + flagSet.BoolVar(&read, "read", false, "Print the current secret stored in the keyring for this service to stdout and exit") + flagSet.BoolVar(&clear, "clear", false, "Clear the secret stored in the keyring for this service and exit") + + if err := flagSet.Parse(args); err != nil { + fatalFmt("parsing flags for setup-keyring: %v", err) + } + + if flagSet.NArg() != 1 { + flagSet.Usage() + } + + service := flagSet.Arg(0) + + if read && clear { + fatalFmt("cannot specify both --read and --clear") + } + if read { + secret, err := keyring.Get(service, "") + if err != nil { + fatalFmt("reading secret from keyring: %v", err) + } + fmt.Print(secret) + return + } + if clear { + if err := keyring.Delete(service, ""); err != nil { + fatalFmt("deleting secret from keyring: %v", err) + } + fmt.Printf("Cleared secret %s\n", service) + return + } + + var sourceFile io.Reader + if len(sourceFilePath) > 0 { + // the credential-helper process changes it's own working directory + // during setup. + // This remapping is necessary to find original, relative paths. + sourceFilePath = locate.RemapToOriginalWorkingDirectory(sourceFilePath) + var openErr error + sourceFile, openErr = os.OpenFile(sourceFilePath, os.O_RDONLY, 0) + if openErr != nil { + fatalFmt("opening source file %s: %v", sourceFilePath, openErr) + } + } else { + // use stdin as source + sourceFile = os.Stdin + } + + secret, err := io.ReadAll(sourceFile) + if err != nil { + sourceName := sourceFilePath + if sourceName == "" { + sourceName = "stdin" + fmt.Fprintf(os.Stderr, "Reading secret from stdin.\n") + } + fatalFmt("reading secret from %s: %v", sourceName, err) + } + + if err := keyring.Set(service, "", string(secret)); err != nil { + fatalFmt("storing secret in keyring: %v", err) + } + + fmt.Printf("Stored secret %s\n", service) +} + +func fatalFmt(format string, args ...any) { + if !strings.HasSuffix(format, "\n") { + format += "\n" + } + fmt.Fprintf(os.Stderr, format, args...) + os.Exit(1) +} From 0815c4535d892c418c02e8beec7da244edb9e267 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Sat, 22 Feb 2025 18:52:16 +0100 Subject: [PATCH 5/8] add setup-uri command --- api/api.go | 2 +- authenticate/gcs/gcs.go | 23 +++++++++ authenticate/github/github.go | 29 +++++++++++ bzl/private/source_files/BUILD.bazel | 1 + cmd/internal/util/BUILD.bazel | 19 +++++++ cmd/internal/util/util.go | 35 +++++++++++++ cmd/root/BUILD.bazel | 1 + cmd/root/root.go | 38 +++++--------- cmd/setup/BUILD.bazel | 9 +++- cmd/setup/uri.go | 76 ++++++++++++++++++++++++++++ 10 files changed, 207 insertions(+), 26 deletions(-) create mode 100644 cmd/internal/util/BUILD.bazel create mode 100644 cmd/internal/util/util.go create mode 100644 cmd/setup/uri.go diff --git a/api/api.go b/api/api.go index 2396a21..38e7a64 100644 --- a/api/api.go +++ b/api/api.go @@ -74,7 +74,7 @@ type Cache interface { // URISetupper is an optional interface that can be implemented by helpers to perform setup for a given URI. type URISetupper interface { - SetupInstructionsForURI(uri string) string + SetupInstructionsForURI(ctx context.Context, uri string) string } var CacheMiss = errors.New("cache miss") diff --git a/authenticate/gcs/gcs.go b/authenticate/gcs/gcs.go index 5fba761..43a97f3 100644 --- a/authenticate/gcs/gcs.go +++ b/authenticate/gcs/gcs.go @@ -3,6 +3,7 @@ package gcs import ( "context" "errors" + "fmt" "net/url" "time" @@ -19,6 +20,28 @@ func (g *GCS) CacheKey(req api.GetCredentialsRequest) string { return "https://storage.googleapis.com/" } +func (g *GCS) SetupInstructionsForURI(ctx context.Context, uri string) string { + return fmt.Sprintf(`%s is a Google Cloud Storage (GCS) url. + +IAM Setup: + + In order to access data from a bucket, you need a Google Cloud user- or service account with read access to the objects you want to access (storage.objects.get). + No other permissions are needed. Refer to Google's documentation for more information: https://cloud.google.com/storage/docs/access-control/iam-permissions + +Authentication Methods: + + Option 1: Using gcloud CLI as a regular user (Recommended) + 1. Install the Google Cloud SDK: https://cloud.google.com/sdk/docs/install + 2. Run: + $ gcloud auth application-default login + 3. Follow the browser prompts to authenticate + + Option 2: Using a Service Account Key, OpenID Connect or other authentication mechanisms + 1. Follow Google's documentation for choosing and setting up your method of choice: https://cloud.google.com/docs/authentication + 2. Ensure your method of choice sets the Application Default Credentials (ADC) environment variable (GOOGLE_APPLICATION_CREDENTIALS): https://cloud.google.com/docs/authentication/provide-credentials-adc + 3. Alternatively, check that the credentials file is in a well-known location ($HOME/.config/gcloud/application_default_credentials.json)`, uri) +} + func (GCS) Resolver(ctx context.Context) (api.Resolver, error) { credentials, err := gauth.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/devstorage.read_only") if err != nil { diff --git a/authenticate/github/github.go b/authenticate/github/github.go index 71b7dd3..7567352 100644 --- a/authenticate/github/github.go +++ b/authenticate/github/github.go @@ -30,6 +30,35 @@ func (g *GitHub) Resolver(ctx context.Context) (api.Resolver, error) { return &GitHubResolver{tokenSource: source}, nil } +func (g *GitHub) SetupInstructionsForURI(ctx context.Context, uri string) string { + return fmt.Sprintf(`%s is a GitHub url. + +The credential helper can be used to download any assets GitHub hosts, including: + + - the git protocol via https + - raw code files (raw.githubusercontent.com////) + - patches (github.com///.patch) + - source tarballs (github.com///archive/refs/tags/v1.2.3.tar.gz) + - release assets (github.com///releases/download/v1.2.3/) + - container images from ghcr.io (doc) + ... and more. + +With credentials, you are also less likely to be blocked by GitHub rate limits, even when accessing public repositories. + +Authentication Methods: + + Option 1: Using the GitHub CLI as a regular user (Recommended) + 1. Install the GitHub CLI (gh): https://github.com/cli/cli#installation + 2. Login via: + $ gh auth login + 3. Follow the browser prompts to authenticate + + Option 2: Authentication using a GitHub App, GitHub Actions Token or Personal Access Token (PAT) + 1. Setup your authentication method of choice + 2. Set the required environment variable (GH_TOKEN or GITHUB_TOKEN) when running Bazel (or other tools that access credential helpers) + 3. Alternatively, add the secret to the system keyring under the gh:github.com key`, uri) +} + // CacheKey returns a cache key for the given request. // For GitHub, the same token can be used for all requests. func (g *GitHub) CacheKey(req api.GetCredentialsRequest) string { diff --git a/bzl/private/source_files/BUILD.bazel b/bzl/private/source_files/BUILD.bazel index 7b35238..99239ef 100644 --- a/bzl/private/source_files/BUILD.bazel +++ b/bzl/private/source_files/BUILD.bazel @@ -24,6 +24,7 @@ release_files = [ "//cmd:all_files", "//cmd/credential-helper:all_files", "//cmd/installer:all_files", + "//cmd/internal/util:all_files", "//cmd/root:all_files", "//cmd/setup:all_files", "//config:all_files", diff --git a/cmd/internal/util/BUILD.bazel b/cmd/internal/util/BUILD.bazel new file mode 100644 index 0000000..7119b17 --- /dev/null +++ b/cmd/internal/util/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "util", + srcs = ["util.go"], + importpath = "github.com/tweag/credential-helper/cmd/internal/util", + visibility = ["//cmd:__subpackages__"], + deps = [ + "//api", + "//config", + "//logging", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//:__subpackages__"], +) diff --git a/cmd/internal/util/util.go b/cmd/internal/util/util.go new file mode 100644 index 0000000..efcc58d --- /dev/null +++ b/cmd/internal/util/util.go @@ -0,0 +1,35 @@ +package util + +import ( + "context" + + "github.com/tweag/credential-helper/api" + "github.com/tweag/credential-helper/config" + "github.com/tweag/credential-helper/logging" +) + +func Configure(ctx context.Context, helperFactory api.HelperFactory, configReader config.ConfigReader, uri string) (context.Context, api.Helper) { + cfg, err := configReader.Read() + if err == nil { + logging.Debugf("found config file and choosing helper from it") + helperFactory = func(uri string) (api.Helper, error) { + helper, helperConfig, err := cfg.FindHelper(uri) + if err != nil { + return nil, err + } + if len(helperConfig) > 0 { + ctx = context.WithValue(ctx, api.HelperConfigKey, helperConfig) + } + return helper, nil + } + } else if err != config.ErrConfigNotFound { + logging.Fatalf("reading config: %v", err) + } + + authenticator, err := helperFactory(uri) + if err != nil { + logging.Fatalf("%v", err) + } + + return ctx, authenticator +} diff --git a/cmd/root/BUILD.bazel b/cmd/root/BUILD.bazel index 433e3c8..f6d94e7 100644 --- a/cmd/root/BUILD.bazel +++ b/cmd/root/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//api", "//cache", "//cmd/installer", + "//cmd/internal/util", "//cmd/setup", "//config", "//logging", diff --git a/cmd/root/root.go b/cmd/root/root.go index 70b6414..2c8eda1 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -16,6 +16,7 @@ import ( "github.com/tweag/credential-helper/api" "github.com/tweag/credential-helper/cache" "github.com/tweag/credential-helper/cmd/installer" + "github.com/tweag/credential-helper/cmd/internal/util" "github.com/tweag/credential-helper/cmd/setup" "github.com/tweag/credential-helper/config" "github.com/tweag/credential-helper/logging" @@ -25,6 +26,7 @@ const usage = `Usage: credential-helper [COMMAND] [ARGS...] Commands: get get credentials in the form of http headers for the uri provided on stdin and print result to stdout (see https://github.com/EngFlow/credential-helper-spec for more information) + setup-uri prints setup instructions for a given uri setup-keyring stores a secret in the system keyring` func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewCache, args []string) { @@ -44,6 +46,8 @@ func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewC switch command { case "get": clientProcess(ctx, helperFactory) + case "setup-uri": + setup.URIProcess(args[2:], helperFactory, config.OSReader{}) case "setup-keyring": setup.KeyringProcess(args[2:]) case "agent-launch": @@ -68,7 +72,7 @@ func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewC // foreground immediately responds to the get command and exits. // If possible, it sends the response to the agent for caching. -func foreground(ctx context.Context, helperFactory api.HelperFactory, cache api.Cache, configReader config.ConfigReader) { +func foreground(ctx context.Context, cache api.Cache, helperFactory api.HelperFactory, configReader config.ConfigReader) { req := api.GetCredentialsRequest{} err := json.NewDecoder(os.Stdin).Decode(&req) @@ -76,27 +80,7 @@ func foreground(ctx context.Context, helperFactory api.HelperFactory, cache api. logging.Fatalf("%v", err) } - cfg, err := configReader.Read() - if err == nil { - logging.Debugf("found config file and choosing helper from it") - helperFactory = func(uri string) (api.Helper, error) { - helper, helperConfig, err := cfg.FindHelper(uri) - if err != nil { - return nil, err - } - if len(helperConfig) > 0 { - ctx = context.WithValue(ctx, api.HelperConfigKey, helperConfig) - } - return helper, nil - } - } else if err != config.ErrConfigNotFound { - logging.Fatalf("reading config: %v", err) - } - - authenticator, err := helperFactory(req.URI) - if err != nil { - logging.Fatalf("%v", err) - } + ctx, authenticator := util.Configure(ctx, helperFactory, configReader, req.URI) cacheKey := authenticator.CacheKey(req) if len(cacheKey) == 0 { @@ -126,7 +110,13 @@ func foreground(ctx context.Context, helperFactory api.HelperFactory, cache api. resp, err = resolver.Get(ctx, req) if err != nil { - logging.Fatalf("%s", err) + var extraMessage string + _, canSetupViaAuthenticator := authenticator.(api.URISetupper) + _, canSetupViaResolver := resolver.(api.URISetupper) + if canSetupViaAuthenticator || canSetupViaResolver { + extraMessage = fmt.Sprintf("\n\nTip: try running the following command for setup instructions:\n $ %s setup-uri %s", os.Args[0], req.URI) + } + logging.Fatalf("%s%s", err, extraMessage) } err = json.NewEncoder(os.Stdout).Encode(resp) @@ -178,7 +168,7 @@ func clientProcess(ctx context.Context, helperFactory api.HelperFactory) { } defer cleanup() - foreground(ctx, helperFactory, cache, config.OSReader{}) + foreground(ctx, cache, helperFactory, config.OSReader{}) } func clientCommandProcess(command string, r io.Reader) { diff --git a/cmd/setup/BUILD.bazel b/cmd/setup/BUILD.bazel index 44366ce..98d6809 100644 --- a/cmd/setup/BUILD.bazel +++ b/cmd/setup/BUILD.bazel @@ -2,11 +2,18 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "setup", - srcs = ["keyring.go"], + srcs = [ + "keyring.go", + "uri.go", + ], importpath = "github.com/tweag/credential-helper/cmd/setup", visibility = ["//visibility:public"], deps = [ "//agent/locate", + "//api", + "//cmd/internal/util", + "//config", + "//logging", "@com_github_zalando_go_keyring//:go-keyring", ], ) diff --git a/cmd/setup/uri.go b/cmd/setup/uri.go new file mode 100644 index 0000000..aa35b5f --- /dev/null +++ b/cmd/setup/uri.go @@ -0,0 +1,76 @@ +package setup + +import ( + "context" + "flag" + "fmt" + "os" + + "github.com/tweag/credential-helper/api" + "github.com/tweag/credential-helper/cmd/internal/util" + "github.com/tweag/credential-helper/config" + "github.com/tweag/credential-helper/logging" +) + +// URIProcess is the entry point for the setup-uri command. +func URIProcess(args []string, helperFactory api.HelperFactory, configReader config.ConfigReader) { + ctx := context.Background() + + flagSet := flag.NewFlagSet("setup-uri", flag.ExitOnError) + flagSet.Usage = func() { + fmt.Fprintf(flagSet.Output(), "Prints setup instructions for a given uri.\n\n") + fmt.Fprintf(flagSet.Output(), "Usage: credential-helper setup-uri [uri]\n") + flagSet.PrintDefaults() + examples := []string{ + "credential-helper setup-uri https://github.com/my-org/project/releases/download/v1.2.3/my-artifact.tar.gz", + "credential-helper setup-uri https://raw.githubusercontent.com/my-org/project/6012...a5de28/file.txt", + "credential-helper setup-uri https://storage.googleapis.com/bucket/path/to/object", + "credential-helper setup-uri https://my-bucket.s3.amazonaws.com/path/to/object", + "credential-helper setup-uri https://org-id.r2.cloudflarestorage.com/bucket/path/to/object", + "credential-helper setup-uri https://index.docker.io/v2/library/hello-world/blobs/sha256:d2c94e...7264ac5a", + } + fmt.Fprintf(flagSet.Output(), "\nExamples:\n") + for _, example := range examples { + fmt.Fprintf(flagSet.Output(), " $ %s\n", example) + } + os.Exit(1) + } + + if err := flagSet.Parse(args); err != nil { + fatalFmt("parsing flags for setup-uri: %v", err) + } + + if flagSet.NArg() != 1 { + flagSet.Usage() + } + + uri := flagSet.Arg(0) + + ctx, authenticator := util.Configure(ctx, helperFactory, configReader, uri) + + var instructionGiver api.URISetupper + + // first try to use the authenticator directly + instructionGiver, ok := authenticator.(api.URISetupper) + + // as a fallback, try to use the resolver + if !ok { + resolver, err := authenticator.Resolver(ctx) + if err != nil { + logging.Fatalf("instantiating resolver: %s", err) + } + + // check if either the resolver or the authenticator provides setup instructions + if resolverInstructionGiver, ok := resolver.(api.URISetupper); ok { + instructionGiver = resolverInstructionGiver + } + } + + if !ok { + fmt.Printf("No setup instructions available for %s\nMaybe the config file is missing or incorrectly configured?", uri) + os.Exit(1) + } + + setupInstructions := instructionGiver.SetupInstructionsForURI(ctx, uri) + fmt.Println(setupInstructions) +} From c8e32b0997b4c12c8eab86f0344c3e774984ebee Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Sat, 22 Feb 2025 19:47:56 +0100 Subject: [PATCH 6/8] add setup instructions for remoteapis --- .../internal/lookupchain/lookupchain.go | 53 ++++++++++++++++++ authenticate/remoteapis/remoteapis.go | 55 ++++++++++++++----- 2 files changed, 93 insertions(+), 15 deletions(-) diff --git a/authenticate/internal/lookupchain/lookupchain.go b/authenticate/internal/lookupchain/lookupchain.go index 381b5df..76411ad 100644 --- a/authenticate/internal/lookupchain/lookupchain.go +++ b/authenticate/internal/lookupchain/lookupchain.go @@ -52,6 +52,25 @@ func (c *LookupChain) Lookup(binding string) (string, error) { return "", fmt.Errorf("no value found for binding %q after querying %v", binding, strings.Join(sourceNames, ", ")) } +func (c *LookupChain) SetupInstructions(binding, meaning string) string { + if len(c.config) == 0 { + return fmt.Sprintf("No sources are configured to look up the secret with binding name %s. Refer to the documentation related to configuration files for more information.", binding) + } + instructions := []string{fmt.Sprintf("Instructions for setting up the secret with binding name %q (%s):", binding, meaning)} + for i, entry := range c.config { + source, err := c.sourceFor(entry) + if err != nil { + instructions = append(instructions, fmt.Sprintf("failed to lookup instuctions for entry %d: %v", i, err)) + continue + } + instruction, ok := source.SetupInstructions(binding) + if ok { + instructions = append(instructions, instruction) + } + } + return strings.Join(instructions, "\n") +} + func (c *LookupChain) sourceFor(entry ConfigEntry) (Source, error) { decoder := json.NewDecoder(bytes.NewReader(entry.RawMessage)) decoder.DisallowUnknownFields() @@ -91,6 +110,7 @@ type ConfigEntry struct { type Source interface { Lookup(binding string) (string, error) Canonicalize() + SetupInstructions(binding string) (string, bool) } type Env struct { @@ -122,6 +142,21 @@ func (e *Env) Canonicalize() { } } +func (e *Env) SetupInstructions(binding string) (string, bool) { + if e.Binding != binding { + return "", false + } + _, success := os.LookupEnv(e.Name) + var status string + if success { + status = "SET" + } else { + status = "NOT SET" + } + + return fmt.Sprintf(" - Export the environment variable %s (status: %s)", e.Name, status), true +} + type Keyring struct { // Source is the name of the source used to look up the secret. // It must be "keyring". @@ -154,6 +189,24 @@ func (k *Keyring) Canonicalize() { } } +func (e *Keyring) SetupInstructions(binding string) (string, bool) { + if e.Binding != binding { + return "", false + } + _, getErr := keyring.Get(e.Service, "") + var status string + if errors.Is(getErr, keyring.ErrNotFound) { + status = "NOT SET" + } else if getErr != nil { + status = "ERROR ACCESSING KEYCHAIN" + } else { + status = "SET" + } + + return fmt.Sprintf(` - Add the secret to the system keyring under the %s service name (status: %s): + $ %s setup-keyring -f secret.txt %s`, e.Service, status, os.Args[0], e.Service), true +} + // Default constructs a partially marshalled Config from a slice of specific config entries. func Default(in []Source) Config { out := make(Config, len(in)) diff --git a/authenticate/remoteapis/remoteapis.go b/authenticate/remoteapis/remoteapis.go index 0cf5e20..a60d48e 100644 --- a/authenticate/remoteapis/remoteapis.go +++ b/authenticate/remoteapis/remoteapis.go @@ -34,6 +34,27 @@ func (g *RemoteAPIs) CacheKey(req api.GetCredentialsRequest) string { return req.URI } +func (g *RemoteAPIs) SetupInstructionsForURI(ctx context.Context, uri string) string { + var lookupChainInstructions string + cfg, err := configFromContext(ctx) + if err == nil { + chain := lookupchain.New(cfg.LookupChain) + lookupChainInstructions = chain.SetupInstructions("default", "secret sent to remote APIs as an authentication token or basic auth credentials") + } else { + lookupChainInstructions = fmt.Sprintf("due to a configuration parsing issue, no further setup instructions are available: %v", err) + } + + var rbeSystemInstructions string + switch { + case strings.HasPrefix(uri, "https://remote.buildbuddy.io/"): + rbeSystemInstructions = `For BuildBuddy, visit https://app.buildbuddy.io/docs/setup/ and copy the secret after "x-buildbuddy-api-key=". Use the header_name "x-buildbuddy-api-key" in the configuration.` + default: + rbeSystemInstructions = "Cannot infer RBE provider based on uri. Skipping provider-specific setup instructions." + } + + return fmt.Sprintf("%s refers to a remote build execution (RBE) system (a gRPC endpoint used for remote execution, remote caching, or related purposes).\n%s\n\n%s", uri, rbeSystemInstructions, lookupChainInstructions) +} + func (RemoteAPIs) Resolver(ctx context.Context) (api.Resolver, error) { return &RemoteAPIs{}, nil } @@ -42,21 +63,7 @@ func (RemoteAPIs) Resolver(ctx context.Context) (api.Resolver, error) { // // https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md#get func (g *RemoteAPIs) Get(ctx context.Context, req api.GetCredentialsRequest) (api.GetCredentialsResponse, error) { - cfg, err := helperconfig.FromContext(ctx, configFragment{ - AuthMethod: "header", - LookupChain: lookupchain.Default([]lookupchain.Source{ - &lookupchain.Env{ - Source: "env", - Name: "CREDENTIAL_HELPER_REMOTEAPIS_SECRET", - Binding: "default", - }, - &lookupchain.Keyring{ - Source: "keyring", - Service: "tweag-credential-helper:remoteapis", - Binding: "default", - }, - }), - }) + cfg, err := configFromContext(ctx) if err != nil { return api.GetCredentialsResponse{}, fmt.Errorf("getting configuration fragment for remotapis helper and url %s: %w", req.URI, err) } @@ -152,3 +159,21 @@ type configFragment struct { // It defaults to the sources "env", "keyring". LookupChain lookupchain.Config `json:"lookup_chain"` } + +func configFromContext(ctx context.Context) (configFragment, error) { + return helperconfig.FromContext(ctx, configFragment{ + AuthMethod: "header", + LookupChain: lookupchain.Default([]lookupchain.Source{ + &lookupchain.Env{ + Source: "env", + Name: "CREDENTIAL_HELPER_REMOTEAPIS_SECRET", + Binding: "default", + }, + &lookupchain.Keyring{ + Source: "keyring", + Service: "tweag-credential-helper:remoteapis", + Binding: "default", + }, + }), + }) +} From 36cc2f6b1b4f9164421f3dc6a5d3284191ff7f3c Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 24 Feb 2025 10:20:27 +0100 Subject: [PATCH 7/8] remoteapis: special-case buildbuddy cloud to work without config file --- authenticate/remoteapis/remoteapis.go | 53 +++++++++++++++++----- helperfactory/fallback/BUILD.bazel | 1 + helperfactory/fallback/fallback_factory.go | 3 ++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/authenticate/remoteapis/remoteapis.go b/authenticate/remoteapis/remoteapis.go index a60d48e..f4ff55c 100644 --- a/authenticate/remoteapis/remoteapis.go +++ b/authenticate/remoteapis/remoteapis.go @@ -35,8 +35,13 @@ func (g *RemoteAPIs) CacheKey(req api.GetCredentialsRequest) string { } func (g *RemoteAPIs) SetupInstructionsForURI(ctx context.Context, uri string) string { + parsedURL, error := url.Parse(uri) + if error != nil { + parsedURL = &url.URL{} + } + var lookupChainInstructions string - cfg, err := configFromContext(ctx) + cfg, err := configFromContext(ctx, parsedURL) if err == nil { chain := lookupchain.New(cfg.LookupChain) lookupChainInstructions = chain.SetupInstructions("default", "secret sent to remote APIs as an authentication token or basic auth credentials") @@ -63,16 +68,6 @@ func (RemoteAPIs) Resolver(ctx context.Context) (api.Resolver, error) { // // https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md#get func (g *RemoteAPIs) Get(ctx context.Context, req api.GetCredentialsRequest) (api.GetCredentialsResponse, error) { - cfg, err := configFromContext(ctx) - if err != nil { - return api.GetCredentialsResponse{}, fmt.Errorf("getting configuration fragment for remotapis helper and url %s: %w", req.URI, err) - } - chain := lookupchain.New(cfg.LookupChain) - secret, err := chain.Lookup("default") - if err != nil { - return api.GetCredentialsResponse{}, err - } - parsedURL, error := url.Parse(req.URI) if error != nil { return api.GetCredentialsResponse{}, error @@ -111,6 +106,17 @@ func (g *RemoteAPIs) Get(ctx context.Context, req api.GetCredentialsRequest) (ap case REMOTE_EXECUTION_V2_EXECUTION: } + cfg, err := configFromContext(ctx, parsedURL) + if err != nil { + return api.GetCredentialsResponse{}, fmt.Errorf("getting configuration fragment for remotapis helper and url %s: %w", req.URI, err) + } + + chain := lookupchain.New(cfg.LookupChain) + secret, err := chain.Lookup("default") + if err != nil { + return api.GetCredentialsResponse{}, err + } + headerName := cfg.HeaderName secretEncoding := func(secret string) string { // by default, the secret is directly used as a header value @@ -160,7 +166,11 @@ type configFragment struct { LookupChain lookupchain.Config `json:"lookup_chain"` } -func configFromContext(ctx context.Context) (configFragment, error) { +func configFromContext(ctx context.Context, uri *url.URL) (configFragment, error) { + if cfg, ok := wellKnownServices[uri.Host]; ok { + return cfg, nil + } + return helperconfig.FromContext(ctx, configFragment{ AuthMethod: "header", LookupChain: lookupchain.Default([]lookupchain.Source{ @@ -177,3 +187,22 @@ func configFromContext(ctx context.Context) (configFragment, error) { }), }) } + +var wellKnownServices = map[string]configFragment{ + "remote.buildbuddy.io": { + AuthMethod: "header", + HeaderName: "x-buildbuddy-api-key", + LookupChain: lookupchain.Default([]lookupchain.Source{ + &lookupchain.Env{ + Source: "env", + Name: "BUILDBUDDY_API_KEY", + Binding: "default", + }, + &lookupchain.Keyring{ + Source: "keyring", + Service: "tweag-credential-helper:buildbuddy_api_key", + Binding: "default", + }, + }), + }, +} diff --git a/helperfactory/fallback/BUILD.bazel b/helperfactory/fallback/BUILD.bazel index 94e5084..b2cf1fd 100644 --- a/helperfactory/fallback/BUILD.bazel +++ b/helperfactory/fallback/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//authenticate/github", "//authenticate/null", "//authenticate/oci", + "//authenticate/remoteapis", "//authenticate/s3", "//logging", ], diff --git a/helperfactory/fallback/fallback_factory.go b/helperfactory/fallback/fallback_factory.go index 9510a6a..748d950 100644 --- a/helperfactory/fallback/fallback_factory.go +++ b/helperfactory/fallback/fallback_factory.go @@ -9,6 +9,7 @@ import ( authenticateGitHub "github.com/tweag/credential-helper/authenticate/github" authenticateNull "github.com/tweag/credential-helper/authenticate/null" authenticateOCI "github.com/tweag/credential-helper/authenticate/oci" + authenticateRemoteAPIs "github.com/tweag/credential-helper/authenticate/remoteapis" authenticateS3 "github.com/tweag/credential-helper/authenticate/s3" "github.com/tweag/credential-helper/logging" ) @@ -33,6 +34,8 @@ func FallbackHelperFactory(rawURL string) (api.Helper, error) { return authenticateGitHub.GitHubContainerRegistry(), nil case strings.HasSuffix(strings.ToLower(u.Host), ".r2.cloudflarestorage.com") && !u.Query().Has("X-Amz-Expires"): return &authenticateS3.R2{}, nil + case strings.EqualFold(u.Host, "remote.buildbuddy.io"): + return &authenticateRemoteAPIs.RemoteAPIs{}, nil // container registries using the default OCI resolver case strings.EqualFold(u.Host, "index.docker.io"): fallthrough From 733840cde8200956453b3659c3b770388ebcf014 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 24 Feb 2025 10:53:29 +0100 Subject: [PATCH 8/8] document remoteapis --- README.md | 26 ++++ docs/lookup_chain.md | 55 ++++++++ docs/providers/remoteapis.md | 240 +++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 docs/lookup_chain.md create mode 100644 docs/providers/remoteapis.md diff --git a/README.md b/README.md index 2a6eef5..8e513e3 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ sequenceDiagram The following providers are supported as of today: +- [Remote Execution & Remote Caching Services](/docs/providers/remoteapis.md) - [AWS S3](/docs/providers/s3.md) - [Cloudflare R2](/docs/providers/r2.md) - [Google Cloud Storage (GCS)](/docs/providers/gcs.md) @@ -142,6 +143,11 @@ Now is a good time to install the credential helper. Simply run `bazel run @twea Alternatively, you can [write custom plugins that are part of your own Bazel workspace and build your own helper][plugins]. Follow the [provider-specific documentation](/docs/providers/) to ensure you can authenticate to the service. +In most cases, you the credential helper can guide you through the required setup for a URL with the following command: + +``` +tools/credential-helper setup-uri +``` You can also look at the [example project](/examples/full/) to see how everything works together. @@ -160,6 +166,7 @@ You can override the config file location using the `$CREDENTIAL_HELPER_CONFIG_F - `.urls[].path`: Path of the url. Matches any path when empty and uses globbing otherwise (a `*` matches any characters). - `.urls[].helper`: Helper to use for this url. Can be one of `s3`, `gcs`, `github`, `oci` or `null`. - `.urls[].config`: Optional helper-specific configuration. Refer to the documentation of the chosen helper for more information. +- `.urls[].config.lookup_chain`: Most helpers support configurable sources for secrets. Consult [the documenation on lookup chains][lookup_chain] for more information. ### Example @@ -182,12 +189,30 @@ You can override the config file location using the `$CREDENTIAL_HELPER_CONFIG_F { "host": "*.oci.acme.corp", "helper": "oci" + }, + { + "host": "bazel-remote.acme.com", + "helper": "remoteapis", + "config": { + "auth_method": "basic_auth", + "lookup_chain": [ + { + "source": "env", + "name": "CREDENTIAL_HELPER_REMOTEAPIS_SECRET" + }, + { + "source": "keyring", + "service": "tweag-credential-helper:remoteapis" + } + ] + } } ] } ``` In this example requests to any path below `https://github.com/tweag/` would use the GitHub helper, any requests to `https://files.acme.corp` that end in `.tar.gz` would use the S3 helper, while any requests to a subdomain of `oci.acme.corp` would use the oci helper. +Additionally, a `baze-remote` instance can be used as a remote cache. ## Environment variables @@ -295,3 +320,4 @@ The agent does not implement additional countermeasures. Consequently, access to [go_duration]: https://pkg.go.dev/time#ParseDuration [plugins]: /docs/plugins.md [bcr]: https://registry.bazel.build/modules/tweag-credential-helper +[lookup_chain]: /docs/lookup_chain.md diff --git a/docs/lookup_chain.md b/docs/lookup_chain.md new file mode 100644 index 0000000..f954f1f --- /dev/null +++ b/docs/lookup_chain.md @@ -0,0 +1,55 @@ +# Secret lookup chains + +The credential helper needs to be flexible when obtaining secrets from the environment. +To support different users with different needs, the configuration file `.tweag-credential-helper.json` allows you to specify where to read secrets from, including the order of preference. + +## Setup steps + +While some servies require specific setup steps, you can login to most services with the following generic steps. +The credential helper tries to explain how to login, based on a given uri. Examples: + +``` +$ credential-helper setup-uri https://github.com/my-org/project/releases/download/v1.2.3/my-artifact.tar.gz +$ credential-helper setup-uri https://raw.githubusercontent.com/my-org/project/6012...a5de28/file.txt +$ credential-helper setup-uri https://storage.googleapis.com/bucket/path/to/object +$ credential-helper setup-uri https://my-bucket.s3.amazonaws.com/path/to/object +$ credential-helper setup-uri https://org-id.r2.cloudflarestorage.com/bucket/path/to/object +$ credential-helper setup-uri https://index.docker.io/v2/library/hello-world/blobs/sha256:d2c94e...7264ac5a +``` + +When using an environment variable, simply `export` the secret value in your shell, like this (replace `SECRET_NAME` with the actual environment variable used for authentication and `secret_value` with the real secret): + +``` +$ export SECRET_NAME=secret_value +``` + +Please note that environment variables easily leak by accident. It is generally more desirable to use a dedicated store for sensitive values. For this purpose, the credential helper can read secrets from the system keyring. +When using the system keyring, you need to know the service name that is used to retrieve the secret. +Simply run the following command to login (replace `secret_value` with the actual secret and `[service-name]` with the name of the secret): + +``` +$ echo -ne "secret_value" | tools/credential-helper setup-keyring [service-name] +``` + + +## Configuration + +Most helpers support lookup chains, unless specifically noted otherwise. +When configuring the helper for a url, you can also define lookup chains. + +When reading sercrets from environment variables, the following options exist: + +- `.urls[].config.lookup_chain[].source`: `"env"` Source of the secret (environment variable) +- `.urls[].config.lookup_chain[].name`: Name of the environment variable to read +- `.urls[].config.lookup_chain[].binding`: Optional binding to a specific secret. If unspecified, it binds to the `"default"` secret. + +When reading sercrets from the system keyring, the following options exist: + +- `.urls[].config.lookup_chain[].source`: `"kering"` Source of the secret (system keyring) +- `.urls[].config.lookup_chain[].service`: Service name used to store the secret in the keyring. +- `.urls[].config.lookup_chain[].binding`: Optional binding to a specific secret. If unspecified, it binds to the `"default"` secret. + +## Secret bindings + +In most cases, you only need a single secret to authenticate. In those cases, the `"default"` binding is used. +For some servies, multiple secrets may be needed. In those cases, the documentation of the servie specifies the name and purpose of a binding. diff --git a/docs/providers/remoteapis.md b/docs/providers/remoteapis.md new file mode 100644 index 0000000..6eac4dc --- /dev/null +++ b/docs/providers/remoteapis.md @@ -0,0 +1,240 @@ +# Remote Build Execution (RBE) services / remote API Authentication + +This document explains how to setup your system for authenticating to remote build execution (RBE) services that are based on [remote api gRPC protocols][remote-apis] for Bazel, Buck2, BuildStream, Reclient, Goma Server, Pants, Please, Recc, Soong, and more. + +Remote APIs include the following: + +- Remote Execution +- Remote Caching +- Build Event UI +- Remote Assets / Remote Downloads + +... and probably more that are offered by a wide range of software and SaaS solutions. + +When using one of the following services, you can directly jump to the matching setup steps: + +- [BuildBuddy Cloud](#section-buildbuddy-cloud) +- [Self-hosted BuildBuddy](#section-buildbuddy-self-hosted) +- [BuildBarn](#section-buildbarn) +- [bazel-remote](#section-bazel-remote) + + +## Configuration + +Configuration depends on the service and authentication mechanism used. +While mTLS cannot be setup using a credential helper, any authentication scheme based on HTTP headers should work. + +The configuration in `.tweag-credential-helper.json` supports the following values: + +- `.urls[].helper`: `"remoteapis"` (name of the helper) +- `.urls[].auth_method`: one of + - `"header"`: Default. Send a HTTP header with the value being the default secret. + - `"basic_auth"` Used by `bazel-remote`. Send the default secret containing username and password (`username:password`) as a basic auth header. +- `.urls[].header_name`: Name of the HTTP header used for authentication. Example: use `"x-buildbuddy_api_key"` for BuildBuddy. +- `.urls[].lookup_chain`: The [lookup chain][lookup_chain] used to find the `default` secret. Defaults to: + ```json + [ + { + "source": "env", + "name": "CREDENTIAL_HELPER_REMOTEAPIS_SECRET", + "binding": "default" + }, + { + "source": "keyring", + "name": "tweag-credential-helper:remoteapis", + "binding": "default" + } + ] + ``` + + +### BuildBuddy Cloud (remote.buildbuddy.io) + +Add to your `.bazelrc`: + +``` +common --credential_helper=remote.buildbuddy.io=%workspace%/tools/credential-helper +``` + +BuildBuddy uses a the `x-buildbuddy-api-key` HTTP header for authentication. +Visit the [setup page][buildbuddy-setup], and copy the secret after `x-buildbuddy-api-key=`. + +If you are not using a configuration file, you can authenticate with an environment variable or a keyring secret: + +- Set `$BUILDBUDDY_API_KEY` to the value of the `x-buildbuddy-api-key` HTTP header. +- Set the `tweag-credential-helper:buildbuddy_api_key` secret to the value of the `x-buildbuddy-api-key` HTTP header: + + ``` + $ echo -ne "$BUILDBUDDY_API_KEY" | tools/credential-helper setup-keyring tweag-credential-helper:buildbuddy_api_key + ``` + +If you need to customize the HTTP header or secret used, read the next section on setting up self-hosted BuildBuddy instead: + +### Self-hosted BuildBuddy + +In the following snippets, we assume that your BuildBuddy instance is hosted under `buildbuddy.acme.com`. Replace this hostname with your own. + +Add to your `.bazelrc`: + +``` +common --credential_helper=buildbuddy.acme.com=%workspace%/tools/credential-helper +``` + +Add to your `.tweag-credential-helper.json`: +```json +{ + "urls": [ + { + "host": "buildbuddy.acme.com", + "helper": "remoteapis", + "config": { + "auth_method": "header", + "header_name": "x-buildbuddy-api-key", + "lookup_chain": [ + { + "source": "env", + "name": "BUILDBUDDY_API_KEY" + }, + { + "source": "keyring", + "service": "tweag-credential-helper:buildbuddy_api_key" + } + ] + } + } + ] +} +``` + +### BuildBarn + +BuildBarn supports a variety of authentication mechanisms specified in the Jsonnet key `authenticationPolicy`. +Only the polcies `jwt` and `remote` can be used to authenticate using HTTP headers (at the time of writing). +In the following snippets, we assume that your BuildBarn instance is hosted under `buildbarn.acme.com`. Replace this hostname with your own. + +#### JWT authentication + +Configure the BuildBarn Jsonnet for `jwt` (more setup needed - setting up and distributing keys is out of the scope of this document). It is also assumed that the user of Bazel already has access to a jwt in the `$BUILDBARN_API_KEY` environment variable, which must be encoded as follows: `Bearer `. + +```Jsonnet +authenticationPolicy: { + jwt: { + jwksFile: "some/file/path.jwks" + ... + } +} +``` + +#### Custom auth middleware / remote auth + +Configure the BuildBarn Jsonnet for `remote` (more setup needed - setting up the authentication middleware is out of the scope of this document). We assume that `x-buildbarn-api-key` is the header forwarded to the authentication middleware. It is also assumed that the user of Bazel already has access to a token in the `$BUILDBARN_API_KEY` environment variable, which must be encoded as-is. +Replace the endpoint address with the address of your custom authentication middleware. + +```Jsonnet +authenticationPolicy: { + remote: { + headers: ["x-buildbarn-api-key"] + endpoint: { + address: "address:port" + ... + } + ... + } +} +``` + +#### Bazel and credential-helper Configuration + +Add to your `.bazelrc`: + +``` +common --credential_helper=buildbarn.acme.com=%workspace%/tools/credential-helper +``` + +Add to your `.tweag-credential-helper.json`: +```json +{ + "urls": [ + { + "host": "buildbarn.acme.com", + "helper": "remoteapis", + "config": { + "auth_method": "header", + "header_name": "Authorization", + "lookup_chain": [ + { + "source": "env", + "name": "BUILDBARN_API_KEY" + }, + { + "source": "keyring", + "service": "tweag-credential-helper:buildbarn_api_key" + } + ] + } + } + ] +} +``` + +When using the system keyring, login with the following command: + +``` +$ echo -ne "$BUILDBARN_API_KEY" | tools/credential-helper setup-keyring tweag-credential-helper:buildbarn_api_key +``` + +### bazel-remote + +The only header-based authentication scheme [bazel-remote][bazel-remote] supports at the time of writing is basic auth (username and password). + +In the following snippets, we assume that your bazel-remote instance is hosted under `bazel-remote.acme.com`. Replace this hostname with your own. +Additionally, we assume that the user already created a `.htpasswd` file under `/etc/bazel-remote/.htpasswd` for bazel-remote that contains credentials for the user. + +Add to your bazel-remote configuration yaml: + +```yaml +htpasswd_file: /etc/bazel-remote/.htpasswd +``` + +Add to your `.bazelrc`: + +``` +common --credential_helper=bazel-remote.acme.com=%workspace%/tools/credential-helper +``` + +Add to your `.tweag-credential-helper.json`: +```json +{ + "urls": [ + { + "host": "bazel-remote.acme.com", + "helper": "remoteapis", + "config": { + "auth_method": "basic_auth", + "lookup_chain": [ + { + "source": "env", + "name": "CREDENTIAL_HELPER_REMOTEAPIS_SECRET" + }, + { + "source": "keyring", + "service": "tweag-credential-helper:remoteapis" + } + ] + } + } + ] +} +``` + +Users can either export the `$CREDENTIAL_HELPER_REMOTEAPIS_SECRET` environment variable, or login with the system keyring using the following command: + +``` +$ echo -ne "username:password" | tools/credential-helper setup-keyring tweag-credential-helper:remoteapis +``` + + +[remote-apis]: https://github.com/bazelbuild/remote-apis +[buildbuddy-setup]: https://app.buildbuddy.io/docs/setup/ +[lookup_chain]: /docs/lookup_chain.md +[bazel-remote]: https://github.com/buchgr/bazel-remote