add support for remote execution APIs

README.md

@@ -50,6 +50,7 @@ sequenceDiagram
 
 The following providers are supported as of today:
 
+- [Remote Execution & Remote Caching Services](/docs/providers/remoteapis.md)
 - [AWS S3](/docs/providers/s3.md)
 - [Cloudflare R2](/docs/providers/r2.md)
 - [Google Cloud Storage (GCS)](/docs/providers/gcs.md)
@@ -142,6 +143,11 @@ Now is a good time to install the credential helper. Simply run `bazel run @twea
 Alternatively, you can [write custom plugins that are part of your own Bazel workspace and build your own helper][plugins].
 
 Follow the [provider-specific documentation](/docs/providers/) to ensure you can authenticate to the service.
+In most cases, you the credential helper can guide you through the required setup for a URL with the following command:
+
+```
+tools/credential-helper setup-uri <URL>
+```
 
 You can also look at the [example project](/examples/full/) to see how everything works together.
 
@@ -160,6 +166,7 @@ You can override the config file location using the `$CREDENTIAL_HELPER_CONFIG_F
 - `.urls[].path`: Path of the url. Matches any path when empty and uses globbing otherwise (a `*` matches any characters).
 - `.urls[].helper`: Helper to use for this url. Can be one of `s3`, `gcs`, `github`, `oci` or `null`.
 - `.urls[].config`: Optional helper-specific configuration. Refer to the documentation of the chosen helper for more information.
+- `.urls[].config.lookup_chain`: Most helpers support configurable sources for secrets. Consult [the documenation on lookup chains][lookup_chain] for more information.
 
 ### Example
 
@@ -182,12 +189,30 @@ You can override the config file location using the `$CREDENTIAL_HELPER_CONFIG_F
     {
       "host": "*.oci.acme.corp",
       "helper": "oci"
+    },
+    {
+      "host": "bazel-remote.acme.com",
+      "helper": "remoteapis",
+      "config": {
+        "auth_method": "basic_auth",
+        "lookup_chain": [
+            {
+                "source": "env",
+                "name": "CREDENTIAL_HELPER_REMOTEAPIS_SECRET"
+            },
+            {
+                "source": "keyring",
+                "service": "tweag-credential-helper:remoteapis"
+            }
+        ]
+      }
     }
   ]
 }
 ```
 
 In this example requests to any path below `https://github.com/tweag/` would use the GitHub helper, any requests to `https://files.acme.corp` that end in `.tar.gz` would use the S3 helper, while any requests to a subdomain of `oci.acme.corp` would use the oci helper.
+Additionally, a `baze-remote` instance can be used as a remote cache.
 
 ## Environment variables
 
@@ -295,3 +320,4 @@ The agent does not implement additional countermeasures. Consequently, access to
 [go_duration]: https://pkg.go.dev/time#ParseDuration
 [plugins]: /docs/plugins.md
 [bcr]: https://registry.bazel.build/modules/tweag-credential-helper
+[lookup_chain]: /docs/lookup_chain.md

agent/locate/locate.go

@@ -17,6 +17,14 @@ import (
 // environment variables to ensure
 // a consistent working environment.
 func SetupEnvironment() error {
+	originalWorkingDirectory, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	if err := os.Setenv(api.OriginalWorkingDirectoryEnv, originalWorkingDirectory); err != nil {
+		return err
+	}
+
 	workspacePath, err := setupWorkspaceDirectory()
 	if err != nil {
 		return err
@@ -111,6 +119,14 @@ func AgentPaths() (string, string) {
 	return socketPath, pidPath
 }
 
+func RemapToOriginalWorkingDirectory(p string) string {
+	if filepath.IsAbs(p) {
+		return p
+	}
+	prefix := os.Getenv(api.OriginalWorkingDirectoryEnv)
+	return filepath.Join(prefix, p)
+}
+
 func tmpDir() string {
 	// In Bazel integration tests we can't (and shouldn't)
 	// touch /tmp

api/api.go

@@ -72,6 +72,11 @@ type Cache interface {
 	Prune(context.Context) error
 }
 
+// URISetupper is an optional interface that can be implemented by helpers to perform setup for a given URI.
+type URISetupper interface {
+	SetupInstructionsForURI(ctx context.Context, uri string) string
+}
+
 var CacheMiss = errors.New("cache miss")
 
 // Environment variable names used by the credential helper.
@@ -90,6 +95,9 @@ const (
 	WorkdirEnv = "CREDENTIAL_HELPER_WORKDIR"
 	// The working directory of Bazel (path containing root module).
 	WorkspaceEnv = "CREDENTIAL_HELPER_WORKSPACE_DIRECTORY"
+	// The cwd of the helper process before it is chdir'd into the workdir.
+	// This is used to resolve relative paths for some CLI commands.
+	OriginalWorkingDirectoryEnv = "CREDENTIAL_HELPER_ORIGINAL_WORKING_DIRECTORY"
 )
 
 // Placeholders in configuration that is expanded automatically.

authenticate/gcs/gcs.go

@@ -3,6 +3,7 @@ package gcs
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/url"
 	"time"
 
@@ -19,6 +20,28 @@ func (g *GCS) CacheKey(req api.GetCredentialsRequest) string {
 	return "https://storage.googleapis.com/"
 }
 
+func (g *GCS) SetupInstructionsForURI(ctx context.Context, uri string) string {
+	return fmt.Sprintf(`%s is a Google Cloud Storage (GCS) url.
+
+IAM Setup:
+
+  In order to access data from a bucket, you need a Google Cloud user- or service account with read access to the objects you want to access (storage.objects.get).
+  No other permissions are needed. Refer to Google's documentation for more information: https://cloud.google.com/storage/docs/access-control/iam-permissions
+
+Authentication Methods:
+
+  Option 1: Using gcloud CLI as a regular user (Recommended)
+    1. Install the Google Cloud SDK: https://cloud.google.com/sdk/docs/install
+    2. Run:
+      $ gcloud auth application-default login
+    3. Follow the browser prompts to authenticate
+
+  Option 2: Using a Service Account Key, OpenID Connect or other authentication mechanisms
+    1. Follow Google's documentation for choosing and setting up your method of choice: https://cloud.google.com/docs/authentication
+    2. Ensure your method of choice sets the Application Default Credentials (ADC) environment variable (GOOGLE_APPLICATION_CREDENTIALS): https://cloud.google.com/docs/authentication/provide-credentials-adc
+    3. Alternatively, check that the credentials file is in a well-known location ($HOME/.config/gcloud/application_default_credentials.json)`, uri)
+}
+
 func (GCS) Resolver(ctx context.Context) (api.Resolver, error) {
 	credentials, err := gauth.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/devstorage.read_only")
 	if err != nil {

authenticate/github/github.go

@@ -30,6 +30,35 @@ func (g *GitHub) Resolver(ctx context.Context) (api.Resolver, error) {
 	return &GitHubResolver{tokenSource: source}, nil
 }
 
+func (g *GitHub) SetupInstructionsForURI(ctx context.Context, uri string) string {
+	return fmt.Sprintf(`%s is a GitHub url.
+
+The credential helper can be used to download any assets GitHub hosts, including:
+
+  - the git protocol via https
+  - raw code files (raw.githubusercontent.com/<org>/<repo>/<commit>/<file>)
+  - patches (github.com/<org>/<repo>/<commit>.patch)
+  - source tarballs (github.com/<org>/<repo>/archive/refs/tags/v1.2.3.tar.gz)
+  - release assets (github.com/<org>/<repo>/releases/download/v1.2.3/<file>)
+  - container images from ghcr.io (doc)
+  ... and more.
+
+With credentials, you are also less likely to be blocked by GitHub rate limits, even when accessing public repositories.
+
+Authentication Methods:
+
+  Option 1: Using the GitHub CLI as a regular user (Recommended)
+    1. Install the GitHub CLI (gh): https://github.com/cli/cli#installation
+    2. Login via:
+      $ gh auth login
+    3. Follow the browser prompts to authenticate
+
+  Option 2: Authentication using a GitHub App, GitHub Actions Token or Personal Access Token (PAT)
+    1. Setup your authentication method of choice
+    2. Set the required environment variable (GH_TOKEN or GITHUB_TOKEN) when running Bazel (or other tools that access credential helpers)
+    3. Alternatively, add the secret to the system keyring under the gh:github.com key`, uri)
+}
+
 // CacheKey returns a cache key for the given request.
 // For GitHub, the same token can be used for all requests.
 func (g *GitHub) CacheKey(req api.GetCredentialsRequest) string {

authenticate/internal/lookupchain/lookupchain.go

@@ -52,6 +52,25 @@ func (c *LookupChain) Lookup(binding string) (string, error) {
 	return "", fmt.Errorf("no value found for binding %q after querying %v", binding, strings.Join(sourceNames, ", "))
 }
 
+func (c *LookupChain) SetupInstructions(binding, meaning string) string {
+	if len(c.config) == 0 {
+		return fmt.Sprintf("No sources are configured to look up the secret with binding name %s. Refer to the documentation related to configuration files for more information.", binding)
+	}
+	instructions := []string{fmt.Sprintf("Instructions for setting up the secret with binding name %q (%s):", binding, meaning)}
+	for i, entry := range c.config {
+		source, err := c.sourceFor(entry)
+		if err != nil {
+			instructions = append(instructions, fmt.Sprintf("failed to lookup instuctions for entry %d: %v", i, err))
+			continue
+		}
+		instruction, ok := source.SetupInstructions(binding)
+		if ok {
+			instructions = append(instructions, instruction)
+		}
+	}
+	return strings.Join(instructions, "\n")
+}
+
 func (c *LookupChain) sourceFor(entry ConfigEntry) (Source, error) {
 	decoder := json.NewDecoder(bytes.NewReader(entry.RawMessage))
 	decoder.DisallowUnknownFields()
@@ -91,6 +110,7 @@ type ConfigEntry struct {
 type Source interface {
 	Lookup(binding string) (string, error)
 	Canonicalize()
+	SetupInstructions(binding string) (string, bool)
 }
 
 type Env struct {
@@ -122,6 +142,21 @@ func (e *Env) Canonicalize() {
 	}
 }
 
+func (e *Env) SetupInstructions(binding string) (string, bool) {
+	if e.Binding != binding {
+		return "", false
+	}
+	_, success := os.LookupEnv(e.Name)
+	var status string
+	if success {
+		status = "SET"
+	} else {
+		status = "NOT SET"
+	}
+
+	return fmt.Sprintf(" - Export the environment variable %s (status: %s)", e.Name, status), true
+}
+
 type Keyring struct {
 	// Source is the name of the source used to look up the secret.
 	// It must be "keyring".
@@ -154,6 +189,24 @@ func (k *Keyring) Canonicalize() {
 	}
 }
 
+func (e *Keyring) SetupInstructions(binding string) (string, bool) {
+	if e.Binding != binding {
+		return "", false
+	}
+	_, getErr := keyring.Get(e.Service, "")
+	var status string
+	if errors.Is(getErr, keyring.ErrNotFound) {
+		status = "NOT SET"
+	} else if getErr != nil {
+		status = "ERROR ACCESSING KEYCHAIN"
+	} else {
+		status = "SET"
+	}
+
+	return fmt.Sprintf(` - Add the secret to the system keyring under the %s service name (status: %s):
+    $ %s setup-keyring -f secret.txt %s`, e.Service, status, os.Args[0], e.Service), true
+}
+
 // Default constructs a partially marshalled Config from a slice of specific config entries.
 func Default(in []Source) Config {
 	out := make(Config, len(in))

authenticate/remoteapis/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "remoteapis",
+    srcs = ["remoteapis.go"],
+    importpath = "github.com/tweag/credential-helper/authenticate/remoteapis",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api",
+        "//authenticate/internal/helperconfig",
+        "//authenticate/internal/lookupchain",
+        "//logging",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(["*"]),
+    visibility = ["//:__subpackages__"],
+)