Merge pull request #44 from fujiwara/scan

Add scan command.

cli.go

@@ -46,30 +46,82 @@ func SetLogLevel(level string) {
 }
 
 type CLI struct {
-	Config   string `help:"Load configuration from FILE" short:"c" default:"ecrm.yaml" env:"ECRM_CONFIG"`
-	LogLevel string `help:"Set log level (debug, info, notice, warn, error)" default:"info" env:"ECRM_LOG_LEVEL"`
-	Format   string `help:"Output format for plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"`
-	Color    bool   `help:"Whether or not to color the output" default:"true" env:"ECRM_COLOR" negatable:""`
-	Version  bool   `help:"Show version"`
-
-	Plan     *PlanCLI     `cmd:"" help:"Scan ECS/Lambda resources and find unused ECR images to delete safety."`
-	Generate *GenerateCLI `cmd:"" help:"Generate ecrm.yaml"`
+	Config      string `help:"Load configuration from FILE" short:"c" default:"ecrm.yaml" env:"ECRM_CONFIG"`
+	LogLevel    string `help:"Set log level (debug, info, notice, warn, error)" default:"info" env:"ECRM_LOG_LEVEL"`
+	Color       bool   `help:"Whether or not to color the output" default:"true" env:"ECRM_COLOR" negatable:""`
+	ShowVersion bool   `help:"Show version." name:"version"`
+
+	Generate *GenerateCLI `cmd:"" help:"Generate a configuration file."`
+	Scan     *ScanCLI     `cmd:"" help:"Scan ECS/Lambda resources. Output image URIs in use."`
+	Plan     *PlanCLI     `cmd:"" help:"Scan ECS/Lambda resources and find unused ECR images that can be deleted safely."`
 	Delete   *DeleteCLI   `cmd:"" help:"Scan ECS/Lambda resources and delete unused ECR images."`
+	Version  struct{}     `cmd:"" default:"1" help:"Show version."`
 
 	command string
 	app     *App
 }
 
+type GenerateCLI struct {
+}
+
+func (c *GenerateCLI) Option() *Option {
+	return &Option{}
+}
+
 type PlanCLI struct {
-	Repository string `short:"r" help:"plan for only images in REPOSITORY" env:"ECRM_REPOSITORY"`
+	PlanOrDelete
 }
 
-type GenerateCLI struct {
+func (c *PlanCLI) Option() *Option {
+	return &Option{
+		OutputFile:   c.Output,
+		Format:       newOutputFormatFrom(c.Format),
+		Scan:         c.Scan,
+		ScannedFiles: c.ScannedFiles,
+		Delete:       false,
+		Repository:   c.Repository,
+	}
 }
 
 type DeleteCLI struct {
-	Force      bool   `help:"force delete images without confirmation" env:"ECRM_FORCE"`
-	Repository string `short:"r" help:"delete only images in REPOSITORY" env:"ECRM_REPOSITORY"`
+	PlanOrDelete
+	Force bool `help:"force delete images without confirmation" env:"ECRM_FORCE"`
+}
+
+func (c *DeleteCLI) Option() *Option {
+	return &Option{
+		OutputFile:   c.Output,
+		Format:       newOutputFormatFrom(c.Format),
+		Scan:         c.Scan,
+		ScannedFiles: c.ScannedFiles,
+		Delete:       true,
+		Force:        c.Force,
+		Repository:   c.Repository,
+	}
+}
+
+type PlanOrDelete struct {
+	OutputCLI
+	Format       string   `help:"Output format of plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"`
+	Scan         bool     `help:"Scan ECS/Lambda resources that in use." default:"true" negatable:"" env:"ECRM_SCAN"`
+	ScannedFiles []string `help:"Files of the scan result. ecrm does not delete images in these files." env:"ECRM_SCANNED_FILES"`
+	Repository   string   `help:"Delete only images in the repository." short:"r" env:"ECRM_REPOSITORY"`
+}
+
+type OutputCLI struct {
+	Output string `help:"File name of the output. The default is STDOUT." short:"o" default:"-" env:"ECRM_OUTPUT"`
+}
+
+type ScanCLI struct {
+	OutputCLI
+}
+
+func (c *ScanCLI) Option() *Option {
+	return &Option{
+		OutputFile: c.Output,
+		Scan:       true,
+		ScanOnly:   true,
+	}
 }
 
 func (app *App) NewCLI() *CLI {
@@ -81,30 +133,25 @@ func (app *App) NewCLI() *CLI {
 }
 
 func (c *CLI) Run(ctx context.Context) error {
-	if c.Version {
-		log.Println(c.app.Version)
-		return nil
-	}
 	color.NoColor = !c.Color
 	SetLogLevel(c.LogLevel)
 	log.Println("[debug] region:", c.app.region)
 
 	switch c.command {
-	case "plan":
-		return c.app.Run(ctx, c.Config, Option{
-			Delete:     false,
-			Repository: c.Plan.Repository,
-			Format:     newOutputFormatFrom(c.Format),
-		})
 	case "generate":
-		return c.app.GenerateConfig(ctx, c.Config, Option{})
+		return c.app.GenerateConfig(ctx, c.Config, c.Generate.Option())
+	case "scan":
+		return c.app.Run(ctx, c.Config, c.Scan.Option())
+	case "plan":
+		return c.app.Run(ctx, c.Config, c.Plan.Option())
 	case "delete":
-		return c.app.Run(ctx, c.Config, Option{
-			Delete:     true,
-			Force:      c.Delete.Force,
-			Repository: c.Delete.Repository,
-			Format:     newOutputFormatFrom(c.Format),
-		})
+		return c.app.Run(ctx, c.Config, c.Delete.Option())
+	case "version":
+		fmt.Printf("ecrm version %s\n", c.app.Version)
+		if !c.ShowVersion {
+			fmt.Println("Run with --help for usage.")
+		}
+		return nil
 	default:
 		return fmt.Errorf("unknown command: %s", c.command)
 	}

config.go

@@ -140,6 +140,7 @@ func (r *RepositoryConfig) IsExpired(at time.Time) bool {
 }
 
 func LoadConfig(path string) (*Config, error) {
+	log.Println("[info] loading config file:", path)
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, err

ecrm.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"sort"
@@ -42,10 +43,28 @@ type App struct {
 }
 
 type Option struct {
-	Delete     bool
-	Force      bool
-	Repository string
-	Format     outputFormat
+	ScanOnly     bool
+	Scan         bool
+	Delete       bool
+	Force        bool
+	Repository   string
+	OutputFile   string
+	Format       outputFormat
+	ScannedFiles []string
+}
+
+func (opt *Option) Validate() error {
+	if len(opt.ScannedFiles) == 0 && !opt.Scan {
+		return fmt.Errorf("no --scanned-files and --no-scan provided. specify at least one")
+	}
+	return nil
+}
+
+func (opt *Option) OutputWriter() (io.WriteCloser, error) {
+	if opt.OutputFile == "" || opt.OutputFile == "-" {
+		return os.Stdout, nil
+	}
+	return os.Create(opt.OutputFile)
 }
 
 func New(ctx context.Context) (*App, error) {
@@ -88,41 +107,89 @@ func (app *App) taskDefinitionFamilies(ctx context.Context) ([]string, error) {
 	return tds, nil
 }
 
-func (app *App) Run(ctx context.Context, path string, opt Option) error {
+func (app *App) Run(ctx context.Context, path string, opt *Option) error {
+	if err := opt.Validate(); err != nil {
+		return err
+	}
+
 	c, err := LoadConfig(path)
 	if err != nil {
 		return err
 	}
 
+	keepImages := make(Images)
+	if len(opt.ScannedFiles) > 0 {
+		for _, f := range opt.ScannedFiles {
+			log.Println("[info] loading scanned image URIs from", f)
+			imgs := make(Images)
+			if err := imgs.LoadFile(f); err != nil {
+				return err
+			}
+			log.Println("[info] loaded", len(imgs), "image URIs")
+			keepImages.Merge(imgs)
+		}
+	}
+	if opt.Scan {
+		imgs, err := app.DoScan(ctx, c, opt)
+		if err != nil {
+			return err
+		}
+		keepImages.Merge(imgs)
+	}
+	log.Println("[info] total", len(keepImages), "image URIs in use")
+	if opt.ScanOnly {
+		w, err := opt.OutputWriter()
+		if err != nil {
+			return fmt.Errorf("failed to open output: %w", err)
+		}
+		defer w.Close()
+		if err := keepImages.Print(w); err != nil {
+			return err
+		}
+		if opt.OutputFile != "" && opt.OutputFile != "-" {
+			log.Println("[info] saved scanned image URIs to", opt.OutputFile)
+		}
+		return nil
+	}
+
+	return app.DoDelete(ctx, c, opt, keepImages)
+}
+
+func (app *App) DoScan(ctx context.Context, c *Config, opt *Option) (Images, error) {
+	log.Println("[info] scanning resources")
 	// collect images in use by ECS tasks / task definitions
 	var taskdefs []taskdef
-	holdImages := make(Images)
+	keepImages := make(Images)
 	if tds, imgs, err := app.scanClusters(ctx, c.Clusters); err != nil {
-		return err
+		return nil, err
 	} else {
 		taskdefs = append(taskdefs, tds...)
-		holdImages.Merge(imgs)
+		keepImages.Merge(imgs)
 	}
 	if tds, err := app.collectTaskdefs(ctx, c.TaskDefinitions); err != nil {
-		return err
+		return nil, err
 	} else {
 		taskdefs = append(taskdefs, tds...)
 	}
 	if imgs, err := app.collectImages(ctx, taskdefs); err != nil {
-		return err
+		return nil, err
 	} else {
-		holdImages.Merge(imgs)
+		keepImages.Merge(imgs)
 	}
 
 	// collect images in use by lambda functions
 	if imgs, err := app.scanLambdaFunctions(ctx, c.LambdaFunctions); err != nil {
-		return err
+		return nil, err
 	} else {
-		holdImages.Merge(imgs)
+		keepImages.Merge(imgs)
 	}
+	return keepImages, nil
+}
 
+func (app *App) DoDelete(ctx context.Context, c *Config, opt *Option, keepImages Images) error {
+	log.Println("[info] finding expired images")
 	// find candidates to delete
-	candidates, err := app.scanRepositories(ctx, c.Repositories, holdImages, opt)
+	candidates, err := app.scanRepositories(ctx, c.Repositories, keepImages, opt)
 	if err != nil {
 		return err
 	}
@@ -155,7 +222,7 @@ func (app *App) collectImages(ctx context.Context, taskdefs []taskdef) (Images,
 		}
 		for _, id := range ids {
 			if images.Add(id, tds) {
-				log.Printf("[info] image %s is in use by taskdef %s", id.Short(), tds)
+				log.Printf("[info] image %s is in use by taskdef %s", id.String(), tds)
 			}
 		}
 	}
@@ -178,9 +245,9 @@ func (app *App) repositories(ctx context.Context) ([]ecrTypes.Repository, error)
 type deletableImageIDs map[RepositoryName][]ecrTypes.ImageIdentifier
 
 // scanRepositories scans repositories and find expired images
-// holdImages is a set of images in use by ECS tasks / task definitions / lambda functions
+// keepImages is a set of images in use by ECS tasks / task definitions / lambda functions
 // so that they are not deleted
-func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, holdImages Images, opt Option) (deletableImageIDs, error) {
+func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, keepImages Images, opt *Option) (deletableImageIDs, error) {
 	idsMaps := make(deletableImageIDs)
 	sums := SummaryTable{}
 	in := &ecr.DescribeRepositoriesInput{}
@@ -206,7 +273,7 @@ func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, h
 			if rc == nil {
 				continue REPO
 			}
-			imageIDs, sum, err := app.unusedImageIdentifiers(ctx, name, rc, holdImages)
+			imageIDs, sum, err := app.unusedImageIdentifiers(ctx, name, rc, keepImages)
 			if err != nil {
 				return nil, err
 			}
@@ -217,7 +284,13 @@ func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, h
 	sort.SliceStable(sums, func(i, j int) bool {
 		return sums[i].Repo < sums[j].Repo
 	})
-	if err := sums.print(os.Stdout, opt.Format); err != nil {
+	log.Printf("[info] output summary to %s as %s", opt.OutputFile, opt.Format)
+	w, err := opt.OutputWriter()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open output: %w", err)
+	}
+	defer w.Close()
+	if err := sums.print(w, opt.Format); err != nil {
 		return nil, err
 	}
 	return idsMaps, nil
@@ -227,7 +300,7 @@ const batchDeleteImageIdsLimit = 100
 const batchGetImageLimit = 100
 
 // DeleteImages deletes images from the repository
-func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, opt Option) error {
+func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, opt *Option) error {
 	if len(ids) == 0 {
 		log.Println("[info] no need to delete images on", repo)
 		return nil
@@ -264,7 +337,7 @@ func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecr
 }
 
 // unusedImageIdentifiers finds image identifiers(by image digests) from the repository.
-func (app *App) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, holdImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) {
+func (app *App) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, keepImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) {
 	sums := NewRepoSummary(repo)
 	images, imageIndexes, sociIndexes, idByTags, err := app.listImageDetails(ctx, repo)
 	if err != nil {
@@ -283,7 +356,7 @@ IMAGE:
 		// Check if the image is in use (digest)
 		imageURISha256 := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s@%s", *d.RegistryId, app.region, *d.RepositoryName, *d.ImageDigest))
 		log.Printf("[debug] checking %s", imageURISha256)
-		if holdImages.Contains(imageURISha256) {
+		if keepImages.Contains(imageURISha256) {
 			log.Printf("[info] %s@%s is in used, keep it", repo, *d.ImageDigest)
 			continue IMAGE
 		}
@@ -296,7 +369,7 @@ IMAGE:
 			}
 			imageURI := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s", *d.RegistryId, app.region, *d.RepositoryName, tag))
 			log.Printf("[debug] checking %s", imageURI)
-			if holdImages.Contains(imageURI) {
+			if keepImages.Contains(imageURI) {
 				log.Printf("[info] image %s:%s is in used, keep it", repo, tag)
 				continue IMAGE
 			}

ecrm.yaml

@@ -11,3 +11,4 @@ repositories:
     expires: 30 days
     keep_tag_patterns:
       - latest
+    keep_count: 3

generate.go

@@ -25,7 +25,7 @@ func nameToPattern(s string) string {
 	return s
 }
 
-func (app *App) GenerateConfig(ctx context.Context, configFile string, opt Option) error {
+func (app *App) GenerateConfig(ctx context.Context, configFile string, opt *Option) error {
 	config := Config{}
 	if err := app.generateClusterConfig(ctx, &config); err != nil {
 		return err

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/fujiwara/logutils v1.1.0
 	github.com/goccy/go-yaml v1.9.4
+	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.20.0
 	github.com/k1LoW/duration v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5

go.sum

@@ -59,6 +59,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
 github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8=
 github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.20.0 h1:wRqHpOeVh3DnenOrPy9xDOLdnLatiGuuNRVelR2gSbg=
 github.com/google/go-containerregistry v0.20.0/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=