diff --git a/cli.go b/cli.go index c4ecbff..eca2c3f 100644 --- a/cli.go +++ b/cli.go @@ -46,30 +46,82 @@ func SetLogLevel(level string) { } type CLI struct { - Config string `help:"Load configuration from FILE" short:"c" default:"ecrm.yaml" env:"ECRM_CONFIG"` - LogLevel string `help:"Set log level (debug, info, notice, warn, error)" default:"info" env:"ECRM_LOG_LEVEL"` - Format string `help:"Output format for plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"` - Color bool `help:"Whether or not to color the output" default:"true" env:"ECRM_COLOR" negatable:""` - Version bool `help:"Show version"` - - Plan *PlanCLI `cmd:"" help:"Scan ECS/Lambda resources and find unused ECR images to delete safety."` - Generate *GenerateCLI `cmd:"" help:"Generate ecrm.yaml"` + Config string `help:"Load configuration from FILE" short:"c" default:"ecrm.yaml" env:"ECRM_CONFIG"` + LogLevel string `help:"Set log level (debug, info, notice, warn, error)" default:"info" env:"ECRM_LOG_LEVEL"` + Color bool `help:"Whether or not to color the output" default:"true" env:"ECRM_COLOR" negatable:""` + ShowVersion bool `help:"Show version." name:"version"` + + Generate *GenerateCLI `cmd:"" help:"Generate a configuration file."` + Scan *ScanCLI `cmd:"" help:"Scan ECS/Lambda resources. Output image URIs in use."` + Plan *PlanCLI `cmd:"" help:"Scan ECS/Lambda resources and find unused ECR images that can be deleted safely."` Delete *DeleteCLI `cmd:"" help:"Scan ECS/Lambda resources and delete unused ECR images."` + Version struct{} `cmd:"" default:"1" help:"Show version."` command string app *App } +type GenerateCLI struct { +} + +func (c *GenerateCLI) Option() *Option { + return &Option{} +} + type PlanCLI struct { - Repository string `short:"r" help:"plan for only images in REPOSITORY" env:"ECRM_REPOSITORY"` + PlanOrDelete } -type GenerateCLI struct { +func (c *PlanCLI) Option() *Option { + return &Option{ + OutputFile: c.Output, + Format: newOutputFormatFrom(c.Format), + Scan: c.Scan, + ScannedFiles: c.ScannedFiles, + Delete: false, + Repository: c.Repository, + } } type DeleteCLI struct { - Force bool `help:"force delete images without confirmation" env:"ECRM_FORCE"` - Repository string `short:"r" help:"delete only images in REPOSITORY" env:"ECRM_REPOSITORY"` + PlanOrDelete + Force bool `help:"force delete images without confirmation" env:"ECRM_FORCE"` +} + +func (c *DeleteCLI) Option() *Option { + return &Option{ + OutputFile: c.Output, + Format: newOutputFormatFrom(c.Format), + Scan: c.Scan, + ScannedFiles: c.ScannedFiles, + Delete: true, + Force: c.Force, + Repository: c.Repository, + } +} + +type PlanOrDelete struct { + OutputCLI + Format string `help:"Output format of plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"` + Scan bool `help:"Scan ECS/Lambda resources that in use." default:"true" negatable:"" env:"ECRM_SCAN"` + ScannedFiles []string `help:"Files of the scan result. ecrm does not delete images in these files." env:"ECRM_SCANNED_FILES"` + Repository string `help:"Delete only images in the repository." short:"r" env:"ECRM_REPOSITORY"` +} + +type OutputCLI struct { + Output string `help:"File name of the output. The default is STDOUT." short:"o" default:"-" env:"ECRM_OUTPUT"` +} + +type ScanCLI struct { + OutputCLI +} + +func (c *ScanCLI) Option() *Option { + return &Option{ + OutputFile: c.Output, + Scan: true, + ScanOnly: true, + } } func (app *App) NewCLI() *CLI { @@ -81,30 +133,25 @@ func (app *App) NewCLI() *CLI { } func (c *CLI) Run(ctx context.Context) error { - if c.Version { - log.Println(c.app.Version) - return nil - } color.NoColor = !c.Color SetLogLevel(c.LogLevel) log.Println("[debug] region:", c.app.region) switch c.command { - case "plan": - return c.app.Run(ctx, c.Config, Option{ - Delete: false, - Repository: c.Plan.Repository, - Format: newOutputFormatFrom(c.Format), - }) case "generate": - return c.app.GenerateConfig(ctx, c.Config, Option{}) + return c.app.GenerateConfig(ctx, c.Config, c.Generate.Option()) + case "scan": + return c.app.Run(ctx, c.Config, c.Scan.Option()) + case "plan": + return c.app.Run(ctx, c.Config, c.Plan.Option()) case "delete": - return c.app.Run(ctx, c.Config, Option{ - Delete: true, - Force: c.Delete.Force, - Repository: c.Delete.Repository, - Format: newOutputFormatFrom(c.Format), - }) + return c.app.Run(ctx, c.Config, c.Delete.Option()) + case "version": + fmt.Printf("ecrm version %s\n", c.app.Version) + if !c.ShowVersion { + fmt.Println("Run with --help for usage.") + } + return nil default: return fmt.Errorf("unknown command: %s", c.command) } diff --git a/config.go b/config.go index ef9d09b..ad42684 100644 --- a/config.go +++ b/config.go @@ -140,6 +140,7 @@ func (r *RepositoryConfig) IsExpired(at time.Time) bool { } func LoadConfig(path string) (*Config, error) { + log.Println("[info] loading config file:", path) f, err := os.Open(path) if err != nil { return nil, err diff --git a/ecrm.go b/ecrm.go index c1f7c93..5cd03bd 100644 --- a/ecrm.go +++ b/ecrm.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log" "os" "sort" @@ -42,10 +43,28 @@ type App struct { } type Option struct { - Delete bool - Force bool - Repository string - Format outputFormat + ScanOnly bool + Scan bool + Delete bool + Force bool + Repository string + OutputFile string + Format outputFormat + ScannedFiles []string +} + +func (opt *Option) Validate() error { + if len(opt.ScannedFiles) == 0 && !opt.Scan { + return fmt.Errorf("no --scanned-files and --no-scan provided. specify at least one") + } + return nil +} + +func (opt *Option) OutputWriter() (io.WriteCloser, error) { + if opt.OutputFile == "" || opt.OutputFile == "-" { + return os.Stdout, nil + } + return os.Create(opt.OutputFile) } func New(ctx context.Context) (*App, error) { @@ -88,41 +107,89 @@ func (app *App) taskDefinitionFamilies(ctx context.Context) ([]string, error) { return tds, nil } -func (app *App) Run(ctx context.Context, path string, opt Option) error { +func (app *App) Run(ctx context.Context, path string, opt *Option) error { + if err := opt.Validate(); err != nil { + return err + } + c, err := LoadConfig(path) if err != nil { return err } + keepImages := make(Images) + if len(opt.ScannedFiles) > 0 { + for _, f := range opt.ScannedFiles { + log.Println("[info] loading scanned image URIs from", f) + imgs := make(Images) + if err := imgs.LoadFile(f); err != nil { + return err + } + log.Println("[info] loaded", len(imgs), "image URIs") + keepImages.Merge(imgs) + } + } + if opt.Scan { + imgs, err := app.DoScan(ctx, c, opt) + if err != nil { + return err + } + keepImages.Merge(imgs) + } + log.Println("[info] total", len(keepImages), "image URIs in use") + if opt.ScanOnly { + w, err := opt.OutputWriter() + if err != nil { + return fmt.Errorf("failed to open output: %w", err) + } + defer w.Close() + if err := keepImages.Print(w); err != nil { + return err + } + if opt.OutputFile != "" && opt.OutputFile != "-" { + log.Println("[info] saved scanned image URIs to", opt.OutputFile) + } + return nil + } + + return app.DoDelete(ctx, c, opt, keepImages) +} + +func (app *App) DoScan(ctx context.Context, c *Config, opt *Option) (Images, error) { + log.Println("[info] scanning resources") // collect images in use by ECS tasks / task definitions var taskdefs []taskdef - holdImages := make(Images) + keepImages := make(Images) if tds, imgs, err := app.scanClusters(ctx, c.Clusters); err != nil { - return err + return nil, err } else { taskdefs = append(taskdefs, tds...) - holdImages.Merge(imgs) + keepImages.Merge(imgs) } if tds, err := app.collectTaskdefs(ctx, c.TaskDefinitions); err != nil { - return err + return nil, err } else { taskdefs = append(taskdefs, tds...) } if imgs, err := app.collectImages(ctx, taskdefs); err != nil { - return err + return nil, err } else { - holdImages.Merge(imgs) + keepImages.Merge(imgs) } // collect images in use by lambda functions if imgs, err := app.scanLambdaFunctions(ctx, c.LambdaFunctions); err != nil { - return err + return nil, err } else { - holdImages.Merge(imgs) + keepImages.Merge(imgs) } + return keepImages, nil +} +func (app *App) DoDelete(ctx context.Context, c *Config, opt *Option, keepImages Images) error { + log.Println("[info] finding expired images") // find candidates to delete - candidates, err := app.scanRepositories(ctx, c.Repositories, holdImages, opt) + candidates, err := app.scanRepositories(ctx, c.Repositories, keepImages, opt) if err != nil { return err } @@ -155,7 +222,7 @@ func (app *App) collectImages(ctx context.Context, taskdefs []taskdef) (Images, } for _, id := range ids { if images.Add(id, tds) { - log.Printf("[info] image %s is in use by taskdef %s", id.Short(), tds) + log.Printf("[info] image %s is in use by taskdef %s", id.String(), tds) } } } @@ -178,9 +245,9 @@ func (app *App) repositories(ctx context.Context) ([]ecrTypes.Repository, error) type deletableImageIDs map[RepositoryName][]ecrTypes.ImageIdentifier // scanRepositories scans repositories and find expired images -// holdImages is a set of images in use by ECS tasks / task definitions / lambda functions +// keepImages is a set of images in use by ECS tasks / task definitions / lambda functions // so that they are not deleted -func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, holdImages Images, opt Option) (deletableImageIDs, error) { +func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, keepImages Images, opt *Option) (deletableImageIDs, error) { idsMaps := make(deletableImageIDs) sums := SummaryTable{} in := &ecr.DescribeRepositoriesInput{} @@ -206,7 +273,7 @@ func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, h if rc == nil { continue REPO } - imageIDs, sum, err := app.unusedImageIdentifiers(ctx, name, rc, holdImages) + imageIDs, sum, err := app.unusedImageIdentifiers(ctx, name, rc, keepImages) if err != nil { return nil, err } @@ -217,7 +284,13 @@ func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, h sort.SliceStable(sums, func(i, j int) bool { return sums[i].Repo < sums[j].Repo }) - if err := sums.print(os.Stdout, opt.Format); err != nil { + log.Printf("[info] output summary to %s as %s", opt.OutputFile, opt.Format) + w, err := opt.OutputWriter() + if err != nil { + return nil, fmt.Errorf("failed to open output: %w", err) + } + defer w.Close() + if err := sums.print(w, opt.Format); err != nil { return nil, err } return idsMaps, nil @@ -227,7 +300,7 @@ const batchDeleteImageIdsLimit = 100 const batchGetImageLimit = 100 // DeleteImages deletes images from the repository -func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, opt Option) error { +func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, opt *Option) error { if len(ids) == 0 { log.Println("[info] no need to delete images on", repo) return nil @@ -264,7 +337,7 @@ func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecr } // unusedImageIdentifiers finds image identifiers(by image digests) from the repository. -func (app *App) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, holdImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) { +func (app *App) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, keepImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) { sums := NewRepoSummary(repo) images, imageIndexes, sociIndexes, idByTags, err := app.listImageDetails(ctx, repo) if err != nil { @@ -283,7 +356,7 @@ IMAGE: // Check if the image is in use (digest) imageURISha256 := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s@%s", *d.RegistryId, app.region, *d.RepositoryName, *d.ImageDigest)) log.Printf("[debug] checking %s", imageURISha256) - if holdImages.Contains(imageURISha256) { + if keepImages.Contains(imageURISha256) { log.Printf("[info] %s@%s is in used, keep it", repo, *d.ImageDigest) continue IMAGE } @@ -296,7 +369,7 @@ IMAGE: } imageURI := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s", *d.RegistryId, app.region, *d.RepositoryName, tag)) log.Printf("[debug] checking %s", imageURI) - if holdImages.Contains(imageURI) { + if keepImages.Contains(imageURI) { log.Printf("[info] image %s:%s is in used, keep it", repo, tag) continue IMAGE } diff --git a/ecrm.yaml b/ecrm.yaml index 913c1c9..71f2434 100644 --- a/ecrm.yaml +++ b/ecrm.yaml @@ -11,3 +11,4 @@ repositories: expires: 30 days keep_tag_patterns: - latest + keep_count: 3 diff --git a/generate.go b/generate.go index fd30cc2..64edb17 100644 --- a/generate.go +++ b/generate.go @@ -25,7 +25,7 @@ func nameToPattern(s string) string { return s } -func (app *App) GenerateConfig(ctx context.Context, configFile string, opt Option) error { +func (app *App) GenerateConfig(ctx context.Context, configFile string, opt *Option) error { config := Config{} if err := app.generateClusterConfig(ctx, &config); err != nil { return err diff --git a/go.mod b/go.mod index 2cef183..b04a525 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/fatih/color v1.17.0 github.com/fujiwara/logutils v1.1.0 github.com/goccy/go-yaml v1.9.4 + github.com/google/go-cmp v0.5.9 github.com/google/go-containerregistry v0.20.0 github.com/k1LoW/duration v1.1.0 github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 7ed53cf..07b4b36 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8= github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.0 h1:wRqHpOeVh3DnenOrPy9xDOLdnLatiGuuNRVelR2gSbg= github.com/google/go-containerregistry v0.20.0/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= diff --git a/images.go b/images.go index 66025a6..f78712c 100644 --- a/images.go +++ b/images.go @@ -1,6 +1,13 @@ package ecrm -import "strings" +import ( + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" +) // ImageURI represents an image URI. type ImageURI string @@ -46,6 +53,36 @@ func (u ImageURI) Short() string { type Images map[ImageURI]set +func (i Images) Print(w io.Writer) error { + m := make([]string, 0, len(i)) + for k := range i { + m = append(m, string(k)) + } + sort.Strings(m) + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(m); err != nil { + return fmt.Errorf("failed to encode image uris: %w", err) + } + return nil +} + +func (i Images) LoadFile(filename string) error { + f, err := os.Open(filename) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + in := []string{} + if err := json.NewDecoder(f).Decode(&in); err != nil { + return fmt.Errorf("failed to decode images: %w", err) + } + for _, u := range in { + i[ImageURI(u)] = newSet(filename) + } + return nil +} + func (i Images) Add(u ImageURI, usedBy string) bool { if _, ok := i[u]; !ok { i[u] = newSet() diff --git a/images_test.go b/images_test.go index e85cb6f..7a0d3af 100644 --- a/images_test.go +++ b/images_test.go @@ -1,9 +1,12 @@ package ecrm_test import ( + "bytes" + "encoding/json" "testing" "github.com/fujiwara/ecrm" + "github.com/google/go-cmp/cmp" ) func TestImageURI(t *testing.T) { @@ -78,3 +81,38 @@ func TestImages(t *testing.T) { } t.Logf("images: %#v", images) } + +func TestLoadImages(t *testing.T) { + images := make(ecrm.Images) + images.Add("9876543210987.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", "baz") + images.Add("0123456789012.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", "baz") + err := images.LoadFile("testdata/images.json") + if err != nil { + t.Fatal(err) + } + if len(images) != 4 { + t.Errorf("unexpected images: %d", len(images)) + } + t.Logf("images: %#v", images) +} + +func TestPrintImages(t *testing.T) { + images := make(ecrm.Images) + images.Add("9876543210987.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", "baz") + images.Add("0123456789012.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", "baz") + b := &bytes.Buffer{} + err := images.Print(b) + if err != nil { + t.Fatal(err) + } + restored := []string{} + if err := json.NewDecoder(b).Decode(&restored); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]string{ + "0123456789012.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", + "9876543210987.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", + }, restored); diff != "" { + t.Errorf("unexpected images: %s", diff) + } +} diff --git a/lambda.go b/lambda.go index 873cbc1..a315968 100644 --- a/lambda.go +++ b/lambda.go @@ -88,7 +88,7 @@ func (app *App) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) (I } log.Println("[debug] ImageUri", u) if images.Add(u, aws.ToString(v.FunctionArn)) { - log.Printf("[info] %s is in use by Lambda function %s:%s", u.Short(), *v.FunctionName, *v.Version) + log.Printf("[info] %s is in use by Lambda function %s:%s", u.String(), *v.FunctionName, *v.Version) } } } diff --git a/set.go b/set.go index f6dd3bf..bb8a315 100644 --- a/set.go +++ b/set.go @@ -2,8 +2,12 @@ package ecrm type set map[string]struct{} -func newSet() set { - return make(map[string]struct{}) +func newSet(members ...string) set { + s := make(map[string]struct{}) + for _, m := range members { + s[m] = struct{}{} + } + return s } func (s set) add(v string) bool { @@ -42,10 +46,7 @@ func (s set) union(o set) set { if o == nil { return s } - u := newSet() - for k := range s { - u.add(k) - } + u := newSet(s.members()...) for k := range o { u.add(k) } diff --git a/testdata/images.json b/testdata/images.json new file mode 100644 index 0000000..05f653c --- /dev/null +++ b/testdata/images.json @@ -0,0 +1,5 @@ +[ + "0123456789012.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:fe668fb9", + "0123456789012.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:bdbfd1f697393756cddf7f8f5726ad80d1e26c37", + "123456789012.dkr.ecr.us-east-1.amazonaws.com/foo/bar@sha256:c5f65c7e2263b3e9ccc9ce7eb1623dc602c45e5e9871decbe0d221b75777bc2d" +]