From 0c196eb99eacd711e0dffffca7113781a806e549 Mon Sep 17 00:00:00 2001 From: Michael Imamura Date: Mon, 2 Dec 2024 13:01:04 -0500 Subject: [PATCH] Add Plugin Runner Toolbox (#371) * Support containers without a shell. This adds a "toolbox" container that, for now, just provides a BusyBox toolset so that the plugin startup script can run. It also provides a debug shell if the container doesn't ship with one. * Colorize stderr in plugin output. This replaces the shell-based plugin wrapper with a statically-compiled Go wrapper which monitors stdout and stderr. The output lines are serialized (i.e., if the plugin writes to stdout and stderr at the same time, the lines will be written separately) and colorized. This sets us up to implement more useful debug tooling such as result linting, logging, timeouts, etc. * Add basic linting of plugin results. * Build the toolkit separately in quit mode. The build process was generating quite a bit of output on every run. * Fix lint check for "truncated" field value. * Be consistent on "toolbox" (not "toolkit"). * Add comment about run-plugin. * Fix truncated comment. --- .editorconfig | 2 +- backend/utilities/plugin_runner/README.md | 15 +- backend/utilities/plugin_runner/plugin.sh | 42 ++-- .../plugin_runner/toolbox/.dockerignore | 3 + .../plugin_runner/toolbox/Dockerfile | 27 +++ .../utilities/plugin_runner/toolbox/README.md | 3 + .../utilities/plugin_runner/toolbox/go.mod | 15 ++ .../utilities/plugin_runner/toolbox/go.sum | 15 ++ .../utilities/plugin_runner/toolbox/lint.go | 55 +++++ .../plugin_runner/toolbox/lint_test.go | 86 ++++++++ .../utilities/plugin_runner/toolbox/main.go | 194 ++++++++++++++++++ 11 files changed, 427 insertions(+), 30 deletions(-) create mode 100644 backend/utilities/plugin_runner/toolbox/.dockerignore create mode 100644 backend/utilities/plugin_runner/toolbox/Dockerfile create mode 100644 backend/utilities/plugin_runner/toolbox/README.md create mode 100644 backend/utilities/plugin_runner/toolbox/go.mod create mode 100644 backend/utilities/plugin_runner/toolbox/go.sum create mode 100644 backend/utilities/plugin_runner/toolbox/lint.go create mode 100644 backend/utilities/plugin_runner/toolbox/lint_test.go create mode 100644 backend/utilities/plugin_runner/toolbox/main.go diff --git a/.editorconfig b/.editorconfig index d38d83ba..a29eadf8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -24,5 +24,5 @@ trim_trailing_whitespace = false indent_style = space indent_size = 2 -[*.{js,jsx,ts,tsx}] +[*.{go,js,jsx,ts,tsx}] indent_style = tab diff --git a/backend/utilities/plugin_runner/README.md b/backend/utilities/plugin_runner/README.md index 963f0d81..67f450c4 100644 --- a/backend/utilities/plugin_runner/README.md +++ b/backend/utilities/plugin_runner/README.md @@ -35,6 +35,10 @@ Run the "gosec" plugin and start a debug shell in the container: ./plugin.sh run gosec ~/git/my-repo /bin/bash ``` +> [!TIP] +> If the container does not have a shell, use `/opt/artemis-plugin-toolbox/bin/sh` which is a barebones BusyBox shell provided by the runner. +> Once launched, don't forget to add `/opt/artemis-plugin-toolbox/bin` to the `PATH`! + ```text ==> Generating configuration for plugin: gosec [+] Creating 2/2 @@ -119,8 +123,17 @@ Either set the `"writable": true` in the plugin's `settings.json` or run the plu Localstack integration is not yet supported, so any plugin that relies on `artemisdb` either needs to individually support a "databaseless" mode or needs to be configured to connect to an external database. +* Using the `/opt/artemis-plugin-toolbox/bin/sh` shell, most command-line tools are missing. + +Add the toolbox directory to your `PATH`: + +```bash +PATH="/opt/artemis-plugin-toolbox/bin:$PATH" +``` + +This directory isn't added to the path by default to avoid interfering with the plugin being run. + ## Current limitations * No built-in localstack integration yet. This mainly affects plugins which use `artemisdb`. * stdout and stderr are combined into a single stream -- this is a limitation of `docker compose run`. The workaround is to use the debug shell to examine stdout vs stderr. -* The debug shell option does not work with container images which lack a shell. diff --git a/backend/utilities/plugin_runner/plugin.sh b/backend/utilities/plugin_runner/plugin.sh index 86de1a5a..e90458a1 100755 --- a/backend/utilities/plugin_runner/plugin.sh +++ b/backend/utilities/plugin_runner/plugin.sh @@ -72,44 +72,21 @@ function init_compose { local plugindir="$TEMPDIR/plugin" mkdir "$plugindir" || return 1 - local plugincmd - case "$runner" in - core) - plugincmd="python /srv/engine/plugins/$plugin/main.py" - ;; - boxed) - plugincmd="/srv/engine/plugins/plugin.sh --quiet -- $plugin" - ;; - *) - echo "Unsupported plugin runner: $runner" >&2 - return 1 - esac - - # Note: We use /bin/sh for the entrypoint scripts since we don't know - # if the containers have Bash available. + # Note: We use sh from the toolbox since we don't know + # if the containers have a shell available. local plugin_entry="$plugindir/entrypoint.sh" echo "--> Generating: $plugin_entry" cat < "$plugin_entry" || return 1 -#!/bin/sh -$plugincmd \ - "\$(cat /opt/artemis-run-plugin/engine-vars.json)" \ - "\$(cat /opt/artemis-run-plugin/images.json)" \ - "\$(cat /opt/artemis-run-plugin/config.json)" -exitcode=\$? -printf "==> Plugin exited with status: %d " "\$exitcode" -if [ "\$exitcode" -eq 0 ]; then - echo '(success)' -else - echo '(failed)' -fi +#!/opt/artemis-plugin-toolbox/bin/sh +exec /opt/artemis-plugin-toolbox/bin/run-plugin $plugin $runner EOD chmod 755 "$plugin_entry" || return 1 local plugin_debug_entry="$plugindir/entrypoint-debug.sh" echo "--> Generating: $plugin_debug_entry" cat < "$plugin_debug_entry" || return 1 -#!/bin/sh +#!/opt/artemis-plugin-toolbox/bin/sh /opt/artemis-run-plugin/entrypoint.sh echo "==> Starting debug shell: ${debug_shell[@]}" echo ' To run the plugin again with the same configuration:' @@ -152,6 +129,10 @@ EOD cat < "$COMPOSEFILE" || return 1 name: artemis-run-plugin services: + toolbox: + image: "artemis/plugin-toolbox:latest" + build: + context: ../toolbox engine: image: "artemis/engine:latest" container_name: engine @@ -173,6 +154,7 @@ services: command: - $entrypoint volumes_from: + - toolbox:ro - engine:ro volumes: - type: bind @@ -235,6 +217,10 @@ function do_run { init_compose "$plugin" "$image" "$target" "$runner" "$writable" \ "${debug_shell[@]}" || return 1 + echo "--> Building toolbox" + docker compose -f "$COMPOSEFILE" build --quiet || return 1 + + echo "--> Launching plugin" docker compose -f "$COMPOSEFILE" run --rm --remove-orphans plugin } diff --git a/backend/utilities/plugin_runner/toolbox/.dockerignore b/backend/utilities/plugin_runner/toolbox/.dockerignore new file mode 100644 index 00000000..4a4631bb --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/.dockerignore @@ -0,0 +1,3 @@ +* +!*.go +!go.* diff --git a/backend/utilities/plugin_runner/toolbox/Dockerfile b/backend/utilities/plugin_runner/toolbox/Dockerfile new file mode 100644 index 00000000..a7dfcaa6 --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.23 AS builder + +WORKDIR /src/app + +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o run-plugin . + +# Distroless always uses the "latest" tag. +# hadolint ignore=DL3007 +FROM gcr.io/distroless/static-debian11:latest + +WORKDIR /opt/artemis-plugin-toolbox/bin + +# Install a statically-compiled BusyBox in the specific location. +COPY --from=busybox:1.37.0-musl /bin/busybox . +RUN ["./busybox", "--install", "."] + +# Install statically-compiled plugin wrapper. +COPY --from=builder /src/app/run-plugin . + +# The container provides the volume; we don't need to run anything. +VOLUME ["/opt/artemis-plugin-toolbox/bin"] +ENTRYPOINT ["/opt/artemis-plugin-toolbox/bin/true"] diff --git a/backend/utilities/plugin_runner/toolbox/README.md b/backend/utilities/plugin_runner/toolbox/README.md new file mode 100644 index 00000000..a1bdec7e --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/README.md @@ -0,0 +1,3 @@ +# Plugin Runner Toolbox + +Provides a common shell and debugging utilities for the plugin runner utility. diff --git a/backend/utilities/plugin_runner/toolbox/go.mod b/backend/utilities/plugin_runner/toolbox/go.mod new file mode 100644 index 00000000..34552bb6 --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/go.mod @@ -0,0 +1,15 @@ +module github.com/warnermedia/artemis/backend/utilities/plugin_runner/toolbox + +go 1.23.3 + +require ( + github.com/fatih/color v1.18.0 + github.com/packntrack/jsonValidator v0.5.2 + golang.org/x/sys v0.25.0 +) + +require ( + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/backend/utilities/plugin_runner/toolbox/go.sum b/backend/utilities/plugin_runner/toolbox/go.sum new file mode 100644 index 00000000..42386d7d --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/go.sum @@ -0,0 +1,15 @@ +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/packntrack/jsonValidator v0.5.2 h1:ZKp6019Ys4L291jCW7fbEXAsirVS0O7NefZ2nN6N9Z0= +github.com/packntrack/jsonValidator v0.5.2/go.mod h1:UuNIDD2Y4e8lwJbUX7761SGZsDT129TqVI+ed9JkMrM= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/backend/utilities/plugin_runner/toolbox/lint.go b/backend/utilities/plugin_runner/toolbox/lint.go new file mode 100644 index 00000000..bea5ee92 --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/lint.go @@ -0,0 +1,55 @@ +package main + +import ( + "errors" + "fmt" + "strings" + "unicode/utf8" + + "github.com/packntrack/jsonValidator" +) + +type LintErrors []error + +func (errs LintErrors) String() string { + var sb strings.Builder + sb.WriteRune('[') + for i, e := range errs { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%#v", e.Error())) + } + sb.WriteRune(']') + return sb.String() +} + +// lint validates the JSON results from the plugin. +func lint(buf []byte) LintErrors { + var retv []error + + if !utf8.Valid(buf) { + retv = append(retv, errors.New("invalid UTF-8")) + return retv + } + + var result struct { + Success *bool `validations:"type=bool;required=true"` + Truncated *bool `validations:"type=bool;required=true"` + Details any `validations:"required=true"` + Errors []string `validations:"type=[]string;required=true"` + } + + errs := jsonValidator.Validate(buf, &result) + retv = append(retv, errs...) + + if result.Truncated != nil && *result.Truncated { + retv = append(retv, jsonValidator.ValidationError{ + Field: "truncated", Message: "Must be false", + }) + } + + //TODO: Validate details based on plugin type. + + return retv +} diff --git a/backend/utilities/plugin_runner/toolbox/lint_test.go b/backend/utilities/plugin_runner/toolbox/lint_test.go new file mode 100644 index 00000000..1e975c35 --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/lint_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "strings" + "testing" +) + +// containsErr checks if there is an error in the list which contains all +// of the specified substrings. +func containsErr(errs []error, substrs ...string) bool { +search: + for _, e := range errs { + for _, substr := range substrs { + if !strings.Contains(e.Error(), substr) { + continue search + } + } + return true + } + return false +} + +func TestInvalidUTF8(t *testing.T) { + actual := lint([]byte{0xff, 123, 125}) + if !containsErr(actual, "invalid UTF-8") { + t.Fatalf("expected UTF-8 error, got %v", actual) + } +} + +func TestNotObject(t *testing.T) { + actual := lint([]byte(`"success"`)) + if !containsErr(actual, "Field json", "invalid format") { + t.Fatalf("expected type error, got %v", actual) + } +} + +func TestInvalidTypes(t *testing.T) { + actual := lint([]byte(`{ + "success": "yes", + "errors": "bar" + }`)) + if !containsErr(actual, "Field success", "invalid format") { + t.Fatalf("expected type error, got %v", actual) + } + if !containsErr(actual, "Field errors", "invalid format") { + t.Fatalf("expected type error, got %v", actual) + } +} + +func TestTruncated(t *testing.T) { + actual := lint([]byte(`{"truncated": true}`)) + if !containsErr(actual, "Field truncated", "Must be false") { + t.Fatalf("expected value error, got %v", actual) + } +} + +func TestRequiredFields(t *testing.T) { + actual := lint([]byte(`{}`)) + if !containsErr(actual, "Field success", "is required") { + t.Fatalf("expected required error, got %v", actual) + } + if !containsErr(actual, "Field truncated", "is required") { + t.Fatalf("expected required error, got %v", actual) + } + if !containsErr(actual, "Field details", "is required") { + t.Fatalf("expected required error, got %v", actual) + } + if !containsErr(actual, "Field errors", "is required") { + t.Fatalf("expected required error, got %v", actual) + } +} + +func TestValid(t *testing.T) { + actual := lint([]byte(`{ + "success": false, + "truncated": false, + "details": [{ + "component": "foo", + "source": "Dockerfile" + }], + "errors": ["failed to scan"] + }`)) + if len(actual) > 0 { + t.Fatalf("expected no errors, got %v", actual) + } +} diff --git a/backend/utilities/plugin_runner/toolbox/main.go b/backend/utilities/plugin_runner/toolbox/main.go new file mode 100644 index 00000000..35d21925 --- /dev/null +++ b/backend/utilities/plugin_runner/toolbox/main.go @@ -0,0 +1,194 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "sync" + + "github.com/fatih/color" + "golang.org/x/sys/unix" +) + +// Root directory of the plugin sources (mounted from engine container). +const pluginRoot = "/srv/engine/plugins" + +// Root directory of the arg files (generated by the plugin runner script). +const argRoot = "/opt/artemis-run-plugin" + +// PluginOutput is the captured output from the plugin execution. +type PluginOutput struct { + Output []byte // Raw stdout from plugin. + ExitCode int + + LintErrors []error +} + +func NewPluginOutput(output []byte, exitCode int) *PluginOutput { + return &PluginOutput{ + Output: output, + ExitCode: exitCode, + + LintErrors: lint(output), + } +} + +// usage prints the command-line help and exits. +func usage() { + fmt.Fprintf(os.Stderr, "Usage: "+ + os.Args[0]+" plugin runner\n") + os.Exit(1) +} + +// mustRenderCmd generates the command and args. +// Aborts on error. +func mustRenderCmd(plugin, runner string) (name string, args []string) { + switch runner { + case "core": + name = "python" // Use system python from PATH. + args = []string{pluginRoot + "/" + plugin + "/main.py"} + case "boxed": + name = pluginRoot + "/plugin.sh" + args = []string{"--quiet", "--", plugin} + default: + log.Fatal("Unsupported plugin runner: " + runner) + } + + // Append the contents of the arg files. + for _, filename := range []string{"engine-vars.json", "images.json", "config.json"} { + path := argRoot + "/" + filename + buf, err := os.ReadFile(path) + if err != nil { + log.Fatalf("Unable to read arg file %s: %v", path, err) + } + args = append(args, string(buf)) + } + + return +} + +// run executes the plugin. +// stdout and stderr from the plugin are colorized. +// Returns the exitcode. +func run(ctx context.Context, name string, args []string) (*PluginOutput, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdin = nil + outPipe, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to open stdout pipe: %w", err) + } + errPipe, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to open stderr pipe: %w", err) + } + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to execute plugin: %w", err) + } + + // Capture lines from stdout and stderr and colorize them. + // Since we colorize the lines, we use a channel to make sure the + // lines are written sequentially and not on top of each other. + // Initial capacity of the output buffer is anticipating a large + // result. + output := make([]byte, 0, 128*1024) + outchan := make(chan string) + var wg sync.WaitGroup + wg.Add(2) + go func() { + scanner := bufio.NewScanner(outPipe) + for scanner.Scan() { + // Capture JSON for processing. + buf := scanner.Bytes() + output = append(output, buf...) + + outchan <- string(buf) + } + wg.Done() + }() + go func() { + scanner := bufio.NewScanner(errPipe) + for scanner.Scan() { + outchan <- color.RedString(scanner.Text()) + } + wg.Done() + }() + go func() { + wg.Wait() + close(outchan) + }() + for line := range outchan { + fmt.Println(line) + } + + if cmdErr := cmd.Wait(); cmdErr != nil { + exitcode := cmd.ProcessState.ExitCode() + + var exitErr *exec.ExitError + if errors.As(cmdErr, &exitErr) && exitcode != -1 { + // Normal process exit (with potential non-zero exit code). + return NewPluginOutput(output, exitcode), nil + } else { + // Other error (e.g. IO interrupt, terminated by signal). + return nil, cmdErr + } + } + return NewPluginOutput(output, cmd.ProcessState.ExitCode()), nil +} + +// installTerminateHandler sets up the handlers for termination signals +// such as Ctrl-C. +func installTerminateHandler(ctx context.Context) context.Context { + retv, cancelFn := context.WithCancel(ctx) + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, unix.SIGTERM, unix.SIGINT, unix.SIGHUP, unix.SIGQUIT) + go func() { + s := <-sigChan + log.Printf("Aborting: %v", s) + // The plugin process, if running, will be killed. + cancelFn() + }() + + return retv +} + +func reportStatus(exitCode int) { + fmt.Print(color.HiCyanString( + fmt.Sprintf("==> Plugin exited with status: %d ", exitCode))) + if exitCode == 0 { + color.HiGreen("(success)") + } else { + color.HiRed("(failed)") + } +} + +func reportLintErrors(errs []error) { + for _, err := range errs { + fmt.Print(color.HiRedString("--> Error: ")) + color.HiYellow(err.Error()) + } +} + +func main() { + if len(os.Args) < 3 { + usage() + } + plugin := os.Args[1] + runner := os.Args[2] + + ctx := installTerminateHandler(context.Background()) + + name, args := mustRenderCmd(plugin, runner) + output, err := run(ctx, name, args) + if err != nil { + log.Fatalf("Error running plugin: %v", err) + } + + reportStatus(output.ExitCode) + reportLintErrors(output.LintErrors) +}