From ad5adf7b0d2fe4222ad36ad78253259efacd5765 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Mon, 2 Dec 2024 23:07:14 +0100 Subject: [PATCH] revamp path handling chdir into workdir to avoid long paths for unix domain sockets --- agent/locate/locate.go | 204 ++++++++++++------ api/api.go | 25 ++- cmd/root/root.go | 18 +- .../customized/helper/cache/sqlitecache.go | 27 +-- installer/installer.go | 9 +- tools/credential-helper | 21 +- 6 files changed, 178 insertions(+), 126 deletions(-) diff --git a/agent/locate/locate.go b/agent/locate/locate.go index 1ba4cfa..dd69808 100644 --- a/agent/locate/locate.go +++ b/agent/locate/locate.go @@ -6,111 +6,175 @@ import ( "os" "path" "runtime" + "strings" "github.com/tweag/credential-helper/api" ) -func Base() (string, error) { - if base, ok := os.LookupEnv(api.CredentialHelperInstallBase); ok { - return base, nil +// SetupEnvironment has to be called early in +// the agent and client process. +// It changes the working directory and exports +// environment variables to ensure +// a consistent working environment. +func SetupEnvironment() error { + workspacePath, err := setupWorkspaceDirectory() + if err != nil { + return err } - // In Bazel integration tests we can't (and shouldn't) - // touch the user's home directory. - // Instead, we operate under $TEST_TMPDIR - var cacheDir string - var err error - cacheDir, ok := os.LookupEnv("TEST_TMPDIR") - if !ok { - // On a normal run, we want to operate in $HOME/.cache (or $XDG_CACHE_HOME) - cacheDir, err = os.UserCacheDir() - if err != nil { - return "", err - } + workdirPath, err := setupWorkdir(workspacePath) + if err != nil { + return err } - workspaceDirectory, err := workspaceDirectory() - if err != nil { - return "", err + if err := os.MkdirAll(workdirPath, os.ModePerm); err != nil { + return err } - return path.Join(cacheDir, "tweag-credential-helper", installBasePathComponent(workspaceDirectory)), nil + return os.Chdir(workdirPath) } -func Bin() (string, error) { - base, err := Base() - if err != nil { - return "", err +func setupWorkspaceDirectory() (string, error) { + // try helper-specific workspace directory env var + workspacePath, haveWorkspacePath := os.LookupEnv(api.WorkspaceEnv) + if haveWorkspacePath { + return workspacePath, nil } - return path.Join(base, "bin"), nil -} -func Run() (string, error) { - base, err := Base() - if err != nil { - return "", err + // maybe we are running under Bazel? + // in that case $BUILD_WORKSPACE_DIRECTORY is the root of the workspace + workspacePath, haveWorkspacePath = os.LookupEnv("BUILD_WORKSPACE_DIRECTORY") + + if !haveWorkspacePath { + // as a last resort + // assume that current working directory + // is the Bazel workspace + var lookupErr error + workspacePath, lookupErr = os.Getwd() + if lookupErr != nil { + return "", lookupErr + } } - return path.Join(base, "run"), nil + return workspacePath, os.Setenv(api.WorkspaceEnv, workspacePath) } -func CredentialHelper() (string, error) { - if path, ok := os.LookupEnv(api.CredentialHelperBin); ok { - return path, nil +func setupWorkdir(workspacePath string) (string, error) { + // try helper-specifc workdir directory env var + workdirPath, haveWorkdirPath := os.LookupEnv(api.WorkdirEnv) + if haveWorkdirPath { + return workdirPath, nil } - bin, err := Bin() - if err != nil { - return "", err + // assume that helper workdir + // is ${cache_dir}/tweag-credential-helper/${workdir_hash} + cacheDir := cacheDir() + workdirPath = path.Join(cacheDir, "tweag-credential-helper", workdirHash(workspacePath)) + return workdirPath, os.Setenv(api.WorkdirEnv, workdirPath) + +} + +func LookupPathEnv(key, fallback string, shortPath bool) string { + unexpanded, ok := os.LookupEnv(key) + if !ok { + unexpanded = fallback } + return expandPath(unexpanded, shortPath) +} + +func Workdir() string { + return os.Getenv(api.WorkdirEnv) +} + +func Bin() string { + return path.Join(Workdir(), "bin") +} + +func Run() string { + return path.Join(Workdir(), "run") +} + +func CredentialHelper() string { filename := "credential-helper" if runtime.GOOS == "windows" { filename += ".exe" } - return path.Join(bin, filename), nil + + return LookupPathEnv(api.CredentialHelperBin, path.Join("%workdir%", "bin", filename), false) } -func AgentPaths() (string, string, error) { - socketPath, haveSocketPathFromEnv := os.LookupEnv(api.AgentSocketPath) - pidPath, havePidPathFromEnv := os.LookupEnv(api.AgentPidPath) - run, err := Run() - if err != nil { - return "", "", err +func AgentPaths() (string, string) { + socketPath := LookupPathEnv(api.AgentSocketPath, path.Join("%workdir%", "run", "agent.sock"), true) + pidPath := LookupPathEnv(api.AgentPidPath, path.Join("%workdir%", "run", "agent.pid"), false) + + return socketPath, pidPath +} + +func tmpDir() string { + // In Bazel integration tests we can't (and shouldn't) + // touch /tmp + // Instead, we operate under $TEST_TMPDIR + if cacheDir, ok := os.LookupEnv("TEST_TMPDIR"); ok { + return cacheDir } - if !haveSocketPathFromEnv { - socketPath = path.Join(run, "agent.sock") - if len(socketPath) >= 108 { - // In many environments - // we are not allowed to use - // a socket path longer than 108 bytes - // - // in those cases, fall back to a unique - // abstract uds (prefixed with @ in Go) - workspaceDirectory, err := workspaceDirectory() - if err != nil { - return "", "", err - } - socketPath = "@" + installBasePathComponent(workspaceDirectory) - } + return os.TempDir() +} +func cacheDir() string { + // In Bazel integration tests we can't (and shouldn't) + // touch the user's home directory. + // Instead, we operate under $TEST_TMPDIR + if cacheDir, ok := os.LookupEnv("TEST_TMPDIR"); ok { + return cacheDir } - if !havePidPathFromEnv { - pidPath = path.Join(run, "agent.pid") + // On a normal run, we want to operate in $HOME/.cache (or $XDG_CACHE_HOME) + cacheDir, err := os.UserCacheDir() + if err != nil { + return tmpDir() } + return cacheDir +} - return socketPath, pidPath, nil +func homeDir() string { + if home, err := os.UserHomeDir(); err == nil { + return home + } + // if we are in an environment where the current + // user doesn't have a resolvable + // home dir, it is probably safer to write to a temp + // dir instead. + return tmpDir() } -func installBasePathComponent(workspaceDirectory string) string { +func workdirHash(workspaceDirectory string) string { return fmt.Sprintf("%x", md5.Sum([]byte(workspaceDirectory))) } -func workspaceDirectory() (string, error) { - workspaceDirectory, ok := os.LookupEnv("BUILD_WORKSPACE_DIRECTORY") - if ok { - return workspaceDirectory, nil +func expandPath(input string, shortPath bool) string { + var prefix, suffix string + var canShortPath bool + switch { + case strings.HasPrefix(input, api.PlaceholderWorkdir): + prefix = os.Getenv(api.WorkdirEnv) + suffix = strings.TrimPrefix(input, api.PlaceholderWorkdir) + // we know that every process chdir's into + // the workdir early, so we can omit the prefix safely. + canShortPath = true + case strings.HasPrefix(input, api.PlaceholderWorkspaceDir): + prefix = os.Getenv(api.WorkspaceEnv) + suffix = strings.TrimPrefix(input, api.PlaceholderWorkspaceDir) + case strings.HasPrefix(input, api.PlaceholderTmpdir): + prefix = tmpDir() + suffix = strings.TrimPrefix(input, api.PlaceholderTmpdir) + case strings.HasPrefix(input, api.PlaceholderCachedir): + prefix = cacheDir() + suffix = strings.TrimPrefix(input, api.PlaceholderCachedir) + case strings.HasPrefix(input, api.PlaceholderHomedir): + prefix = homeDir() + suffix = strings.TrimPrefix(input, api.PlaceholderHomedir) + default: + return input } - workspaceDirectory, err := os.Getwd() - if err != nil { - return "", err + if shortPath && canShortPath { + return path.Join(".", suffix) } - return workspaceDirectory, nil + return path.Join(prefix, suffix) } diff --git a/api/api.go b/api/api.go index 0af36f3..386890e 100644 --- a/api/api.go +++ b/api/api.go @@ -76,12 +76,25 @@ var CacheMiss = errors.New("cache miss") // Environment variable names used by the credential helper. const ( - Standalone = "CREDENTIAL_HELPER_STANDALONE" - CredentialHelperInstallBase = "CREDENTIAL_HELPER_INSTALL_BASE" - CredentialHelperBin = "CREDENTIAL_HELPER_BIN" - AgentSocketPath = "CREDENTIAL_HELPER_AGENT_SOCKET" - AgentPidPath = "CREDENTIAL_HELPER_AGENT_PID" - LogLevelEnv = "CREDENTIAL_HELPER_LOGGING" + Standalone = "CREDENTIAL_HELPER_STANDALONE" + CredentialHelperBin = "CREDENTIAL_HELPER_BIN" + AgentSocketPath = "CREDENTIAL_HELPER_AGENT_SOCKET" + AgentPidPath = "CREDENTIAL_HELPER_AGENT_PID" + LogLevelEnv = "CREDENTIAL_HELPER_LOGGING" + // The working directory for the agent and client process. + // On startup, we chdir into it. + WorkdirEnv = "CREDENTIAL_HELPER_WORKDIR" + // The working directory of Bazel (path containing root module). + WorkspaceEnv = "CREDENTIAL_HELPER_WORKSPACE_DIRECTORY" +) + +// Placeholders in configuration that is expanded automatically. +const ( + PlaceholderWorkdir = "%workdir%" + PlaceholderWorkspaceDir = "%workspace%" + PlaceholderTmpdir = "%tmp%" + PlaceholderCachedir = "%cache%" + PlaceholderHomedir = "~" ) // HelperFactory chooses a credential helper (like s3, gcs, github, ...) based on the raw uri. diff --git a/cmd/root/root.go b/cmd/root/root.go index 7fe4f4a..57332ab 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -26,6 +26,9 @@ func Run(ctx context.Context, helperFactory api.HelperFactory, newCache api.NewC os.Exit(1) } setLogLevel() + if err := locate.SetupEnvironment(); err != nil { + logging.Fatalf("setting up process environment: %v", err) + } command := os.Args[1] switch command { case "get": @@ -121,10 +124,7 @@ func launchOrConnectAgent() (api.Cache, func() error, error) { logging.Debugf("launched agent") - sockPath, _, err := locate.AgentPaths() - if err != nil { - return nil, func() error { return nil }, err - } + sockPath, _ := locate.AgentPaths() logging.Debugf("connecting to agent at %s", sockPath) socketCache, err := cache.NewSocketCache(sockPath, time.Second) if err != nil { @@ -148,10 +148,7 @@ func clientProcess(ctx context.Context, helperFactory api.HelperFactory) { } func clientCommandProcess(command string, r io.Reader) { - socketPath, _, err := locate.AgentPaths() - if err != nil { - logging.Fatalf("%v", err) - } + socketPath, _ := locate.AgentPaths() conn, err := agent.NewAgentCommandClient(socketPath) if err != nil { if command == api.AgentRequestShutdown { @@ -186,10 +183,7 @@ func agentProcess(ctx context.Context, newCache api.NewCache) { logging.Fatalf("running as agent is not supported in standalone mode") } - sockPath, pidPath, err := locate.AgentPaths() - if err != nil { - logging.Fatalf("%v", err) - } + sockPath, pidPath := locate.AgentPaths() service, cleanup, err := agent.NewCachingAgent(sockPath, pidPath, newCache()) if err != nil { logging.Fatalf("%v", err) diff --git a/examples/customized/helper/cache/sqlitecache.go b/examples/customized/helper/cache/sqlitecache.go index d29edd3..85cd0e4 100644 --- a/examples/customized/helper/cache/sqlitecache.go +++ b/examples/customized/helper/cache/sqlitecache.go @@ -22,10 +22,7 @@ type SqliteCache struct { } func NewSqliteCache() api.Cache { - dbFilePath, err := dbPath() - if err != nil { - panic(fmt.Sprintf("failed to find database path: %v", err)) - } + dbFilePath := dbPath() if dbFilePath != ":memory:" { os.MkdirAll(path.Dir(dbFilePath), os.ModePerm) } @@ -119,24 +116,6 @@ func (c *SqliteCache) Prune(_ context.Context) error { return nil } -func dbPath() (string, error) { - dbPath, ok := os.LookupEnv("CREDENTIAL_HELPER_DB_PATH") - run, err := varDir() - if err != nil { - return "", err - } - if !ok { - dbPath = path.Join(run, "database.sqlite") - } - - return dbPath, err -} - -func varDir() (string, error) { - base, err := locate.Base() - if err != nil { - return "", err - } - - return path.Join(base, "var"), nil +func dbPath() string { + return locate.LookupPathEnv("CREDENTIAL_HELPER_DB_PATH", path.Join("%workdir%", "var", "database.sqlite"), false) } diff --git a/installer/installer.go b/installer/installer.go index 313653d..78184eb 100644 --- a/installer/installer.go +++ b/installer/installer.go @@ -17,10 +17,12 @@ func main() { if err != nil { fatalFmt("Failed to find %s: %v", pathFromEnv, err) } - if _, err := os.Stat(path); err != nil { fatalFmt("Failed to stat %s: %v", path, err) } + if err := locate.SetupEnvironment(); err != nil { + fatalFmt("Failed to setup environment %s: %v", path, err) + } destination, err := install(path) if err != nil { fatalFmt("Failed to install %s: %v", path, err) @@ -29,10 +31,7 @@ func main() { } func install(credentialHelperBin string) (string, error) { - destination, err := locate.CredentialHelper() - if err != nil { - return "", err - } + destination := locate.CredentialHelper() if err := os.MkdirAll(path.Dir(destination), 0o755); err != nil { return "", err } diff --git a/tools/credential-helper b/tools/credential-helper index 84c3f6a..1b7d6fd 100755 --- a/tools/credential-helper +++ b/tools/credential-helper @@ -8,9 +8,9 @@ set -o errtrace # export CREDENTIAL_HELPER_STANDALONE=1|0 # If set to 1, the credential helper will run in standalone mode, which means it will not # start or connect to the agent process. -# export CREDENTIAL_HELPER_INSTALL_BASE=/path/to/install_base +# export CREDENTIAL_HELPER_WORKDIR=/path/to/workdir # Path to the directory where the credential helper binary and state (including socket and PID file) -# are stored. If unset, the install base is caclulated based on the user's cache dir and the md5 sum +# are stored. If unset, the workdir is caclulated based on the user's cache dir and the md5 sum # of the current working directory. # export CREDENTIAL_HELPER_BIN=/path/to/credential-helper # Path to the credential helper binary. If not set, the helper will be searched in @@ -26,13 +26,15 @@ set -o errtrace credential_helper_install_command="bazel run @tweag-credential-helper//installer" +export CREDENTIAL_HELPER_WORKSPACE_DIRECTORY="${CREDENTIAL_HELPER_WORKSPACE_DIRECTORY:-$(pwd)}" + # Bazel spawns the credential helper process using the workspace root as working directory. # We abuse this fact to obtain a stable, workspace specific install dir (just like Bazel's output base). -install_base_hash() { +workdir_hash() { if builtin command -v md5 > /dev/null; then - echo -n "$(pwd)" | md5 + echo -n "${CREDENTIAL_HELPER_WORKSPACE_DIRECTORY}" | md5 elif builtin command -v md5sum > /dev/null ; then - local md5_array=($(echo -n "$(pwd)" | md5sum)) + local md5_array=($(echo -n "${CREDENTIAL_HELPER_WORKSPACE_DIRECTORY}" | md5sum)) echo "${md5_array}" else echo "Neither md5 nor md5sum were found in the PATH" >&2 @@ -49,16 +51,17 @@ esac if [ -n "${TEST_TMPDIR}" ]; then cache_dir="${TEST_TMPDIR}" fi -if [ -n "${CREDENTIAL_HELPER_INSTALL_BASE}" ]; then - install_base="${CREDENTIAL_HELPER_INSTALL_BASE}" + +if [ -n "${CREDENTIAL_HELPER_WORKDIR}" ]; then + workdir="${CREDENTIAL_HELPER_WORKDIR}" else - install_base="${cache_dir}/tweag-credential-helper/$(install_base_hash)" + workdir="${cache_dir}/tweag-credential-helper/$(workdir_hash)" fi if [ -n "${CREDENTIAL_HELPER_BIN}" ]; then helper_bin="${CREDENTIAL_HELPER_BIN}" else - helper_bin=${install_base}/bin/credential-helper + helper_bin=${workdir}/bin/credential-helper fi if [ ! -f $helper_bin ]; then