diff --git a/README.md b/README.md index 01e690f..3fea0a6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ecrm -A command line tool for managing ECR repositories. +A command line tool for managing Amazon ECR repositories. ecrm can delete "unused" images safety. @@ -71,13 +71,18 @@ Generate ecrm.yaml ### plan command +`ecrm plan` scans ECS, Lambda and ECR resources in an AWS account and shows summaries of unused images in ECR. + ```console Usage: ecrm plan [flags] Scan ECS/Lambda resources and find unused ECR images to delete safety. Flags: - -r, --repository=STRING plan for only images in REPOSITORY ($ECRM_REPOSITORY) + -o, --output="-" File name of the output. The default is STDOUT ($ECRM_OUTPUT). + --format="table" Output format of plan(table, json) ($ECRM_FORMAT) + --[no-]scan Scan ECS/Lambda resources that in use ($ECRM_SCAN). + -r, --repository=STRING Manage images in the repository only ($ECRM_REPOSITORY). ``` `ecrm plan` shows summaries of unused images in ECR. @@ -94,16 +99,42 @@ $ ecrm plan prod/nginx | 95 (3.7 GB) | -85 (3.3 GB) | 10 (381 MB) ``` +### scan command + +`ecrm scan --output path/to/file` writes the image URIs in use to the file as JSON format. + +The scanned files can be used in the next `ecrm delete` command with `--scanned-files` option. + +The format of the file is a simple JSON array of image URIs. + +```json +[ + "012345678901.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:latest", + "012345678901.dkr.ecr.ap-northeast-1.amazonaws.com/foo/bar:sha256-abcdef1234567890" +] +``` + +You can create scanned files manually as you need. + +If your workload is on platforms that `ecrm plan` does not support (for example, AWS AppRunner, Amazon EKS, etc.), you can use ecrm with the plan file. + ### delete command +`ecrm delete` deletes unused images in ECR repositories. + ```console Usage: ecrm delete [flags] Scan ECS/Lambda resources and delete unused ECR images. Flags: - --force force delete images without confirmation ($ECRM_FORCE) - -r, --repository=STRING delete only images in REPOSITORY ($ECRM_REPOSITORY) + -o, --output="-" File name of the output. The default is STDOUT ($ECRM_OUTPUT). + --format="table" Output format of plan(table, json) ($ECRM_FORMAT) + --[no-]scan Scan ECS/Lambda resources that in use ($ECRM_SCAN). + -r, --repository=STRING Manage images in the repository only ($ECRM_REPOSITORY). + --scanned-files=SCANNED-FILES,... Files of the scan result. ecrm does not delete images in these + files ($ECRM_SCANNED_FILES). + --force force delete images without confirmation ($ECRM_FORCE) ``` ## Notes @@ -132,6 +163,29 @@ See also - [Under the hood: Lazy Loading Container Images with Seekable OCI and AWS Fargate](https://aws.amazon.com/jp/blogs/containers/under-the-hood-lazy-loading-container-images-with-seekable-oci-and-aws-fargate/) - [AWS Fargate Enables Faster Container Startup using Seekable OCI](https://aws.amazon.com/jp/blogs/aws/aws-fargate-enables-faster-container-startup-using-seekable-oci/) +### Multi regions / accounts support. + +`ecrm` supports a single region and an AWS account for each run. + +If your workloads are deployed in multiple regions or accounts, you should run `ecrm scan` for each region or account to collect all image URIs in use. + +Then, you can run `ecrm delete` with the `--scanned-files` option to delete unused images in all regions or accounts. + +For example, your ECR in the `account-a`, and your ECS clusters are deployed in `account-a` and `account-b`. + +At first, you run `ecrm scan` for each accounts. + +```console +$ AWS_PROFILE=account-a ecrm scan --output scan-account-a.json +$ AWS_PROFILE=account-b ecrm scan --output scan-account-b.json +``` + +Now, you can run `ecrm delete` with the `--scanned-files` option to safely delete unused images in all accounts. + +```console +$ AWS_PROFILE=account-a ecrm delete --scanned-files scan-account-a.json,scan-account-b.json +``` + ## Author Copyright (c) 2021 FUJIWARA Shunichiro diff --git a/cli.go b/cli.go index eca2c3f..c262da0 100644 --- a/cli.go +++ b/cli.go @@ -74,18 +74,18 @@ type PlanCLI struct { func (c *PlanCLI) Option() *Option { return &Option{ - OutputFile: c.Output, - Format: newOutputFormatFrom(c.Format), - Scan: c.Scan, - ScannedFiles: c.ScannedFiles, - Delete: false, - Repository: c.Repository, + OutputFile: c.Output, + Format: newOutputFormatFrom(c.Format), + Scan: c.Scan, + Delete: false, + Repository: RepositoryName(c.Repository), } } type DeleteCLI struct { PlanOrDelete - Force bool `help:"force delete images without confirmation" env:"ECRM_FORCE"` + ScannedFiles []string `help:"Files of the scan result. ecrm does not delete images in these files." env:"ECRM_SCANNED_FILES"` + Force bool `help:"force delete images without confirmation" env:"ECRM_FORCE"` } func (c *DeleteCLI) Option() *Option { @@ -96,16 +96,15 @@ func (c *DeleteCLI) Option() *Option { ScannedFiles: c.ScannedFiles, Delete: true, Force: c.Force, - Repository: c.Repository, + Repository: RepositoryName(c.Repository), } } type PlanOrDelete struct { OutputCLI - Format string `help:"Output format of plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"` - Scan bool `help:"Scan ECS/Lambda resources that in use." default:"true" negatable:"" env:"ECRM_SCAN"` - ScannedFiles []string `help:"Files of the scan result. ecrm does not delete images in these files." env:"ECRM_SCANNED_FILES"` - Repository string `help:"Delete only images in the repository." short:"r" env:"ECRM_REPOSITORY"` + Format string `help:"Output format of plan(table, json)" default:"table" enum:"table,json" env:"ECRM_FORMAT"` + Scan bool `help:"Scan ECS/Lambda resources that in use." default:"true" negatable:"" env:"ECRM_SCAN"` + Repository string `help:"Manage images in the repository only." short:"r" env:"ECRM_REPOSITORY"` } type OutputCLI struct { @@ -139,7 +138,7 @@ func (c *CLI) Run(ctx context.Context) error { switch c.command { case "generate": - return c.app.GenerateConfig(ctx, c.Config, c.Generate.Option()) + return c.app.GenerateConfig(ctx, c.Config) case "scan": return c.app.Run(ctx, c.Config, c.Scan.Option()) case "plan": diff --git a/ecrm.go b/ecrm.go index 5cd03bd..b599865 100644 --- a/ecrm.go +++ b/ecrm.go @@ -2,29 +2,17 @@ package ecrm import ( "context" - "encoding/json" "errors" "fmt" - "io" "log" - "os" - "sort" "strings" - "time" "github.com/Songmu/prompter" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/arn" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ecr" ecrTypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/aws/aws-sdk-go-v2/service/ecs" - ecsTypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" - "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/samber/lo" - - oci "github.com/google/go-containerregistry/pkg/v1" - ociTypes "github.com/google/go-containerregistry/pkg/v1/types" ) const ( @@ -36,37 +24,11 @@ var untaggedStr = "__UNTAGGED__" type App struct { Version string + awsCfg aws.Config ecr *ecr.Client - ecs *ecs.Client - lambda *lambda.Client region string } -type Option struct { - ScanOnly bool - Scan bool - Delete bool - Force bool - Repository string - OutputFile string - Format outputFormat - ScannedFiles []string -} - -func (opt *Option) Validate() error { - if len(opt.ScannedFiles) == 0 && !opt.Scan { - return fmt.Errorf("no --scanned-files and --no-scan provided. specify at least one") - } - return nil -} - -func (opt *Option) OutputWriter() (io.WriteCloser, error) { - if opt.OutputFile == "" || opt.OutputFile == "-" { - return os.Stdout, nil - } - return os.Create(opt.OutputFile) -} - func New(ctx context.Context) (*App, error) { cfg, err := awsConfig.LoadDefaultConfig(ctx) if err != nil { @@ -74,242 +36,86 @@ func New(ctx context.Context) (*App, error) { } return &App{ region: cfg.Region, + awsCfg: cfg, ecr: ecr.NewFromConfig(cfg), - ecs: ecs.NewFromConfig(cfg), - lambda: lambda.NewFromConfig(cfg), }, nil } -func (app *App) clusterArns(ctx context.Context) ([]string, error) { - clusters := make([]string, 0) - p := ecs.NewListClustersPaginator(app.ecs, &ecs.ListClustersInput{}) - for p.HasMorePages() { - co, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - clusters = append(clusters, co.ClusterArns...) - } - return clusters, nil -} - -func (app *App) taskDefinitionFamilies(ctx context.Context) ([]string, error) { - tds := make([]string, 0) - p := ecs.NewListTaskDefinitionFamiliesPaginator(app.ecs, &ecs.ListTaskDefinitionFamiliesInput{}) - for p.HasMorePages() { - td, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - log.Println("[debug] task definition families:", td.Families) - tds = append(tds, td.Families...) - } - return tds, nil -} - func (app *App) Run(ctx context.Context, path string, opt *Option) error { if err := opt.Validate(); err != nil { - return err + return fmt.Errorf("invalid option: %w", err) } c, err := LoadConfig(path) if err != nil { - return err + return fmt.Errorf("failed to load config: %w", err) } - keepImages := make(Images) - if len(opt.ScannedFiles) > 0 { - for _, f := range opt.ScannedFiles { - log.Println("[info] loading scanned image URIs from", f) - imgs := make(Images) - if err := imgs.LoadFile(f); err != nil { - return err - } - log.Println("[info] loaded", len(imgs), "image URIs") - keepImages.Merge(imgs) - } + scanner := NewScanner(app.awsCfg) + if err := scanner.LoadFiles(opt.ScannedFiles); err != nil { + return fmt.Errorf("failed to load scanned image URIs: %w", err) } if opt.Scan { - imgs, err := app.DoScan(ctx, c, opt) - if err != nil { - return err + if err := scanner.Scan(ctx, c); err != nil { + return fmt.Errorf("failed to scan: %w", err) } - keepImages.Merge(imgs) } - log.Println("[info] total", len(keepImages), "image URIs in use") + log.Println("[info] total", len(scanner.Images), "image URIs in use") if opt.ScanOnly { - w, err := opt.OutputWriter() - if err != nil { - return fmt.Errorf("failed to open output: %w", err) - } - defer w.Close() - if err := keepImages.Print(w); err != nil { - return err - } - if opt.OutputFile != "" && opt.OutputFile != "-" { - log.Println("[info] saved scanned image URIs to", opt.OutputFile) - } - return nil + return ShowScanResult(scanner, opt) } - return app.DoDelete(ctx, c, opt, keepImages) -} - -func (app *App) DoScan(ctx context.Context, c *Config, opt *Option) (Images, error) { - log.Println("[info] scanning resources") - // collect images in use by ECS tasks / task definitions - var taskdefs []taskdef - keepImages := make(Images) - if tds, imgs, err := app.scanClusters(ctx, c.Clusters); err != nil { - return nil, err - } else { - taskdefs = append(taskdefs, tds...) - keepImages.Merge(imgs) - } - if tds, err := app.collectTaskdefs(ctx, c.TaskDefinitions); err != nil { - return nil, err - } else { - taskdefs = append(taskdefs, tds...) - } - if imgs, err := app.collectImages(ctx, taskdefs); err != nil { - return nil, err - } else { - keepImages.Merge(imgs) + planner := NewPlanner(app.awsCfg) + sums, candidates, err := planner.Plan(ctx, c.Repositories, scanner.Images, opt.Repository) + if err != nil { + return fmt.Errorf("failed to plan: %w", err) } - - // collect images in use by lambda functions - if imgs, err := app.scanLambdaFunctions(ctx, c.LambdaFunctions); err != nil { - return nil, err - } else { - keepImages.Merge(imgs) + if err := ShowSummary(sums, opt); err != nil { + return fmt.Errorf("failed to show summary: %w", err) } - return keepImages, nil -} -func (app *App) DoDelete(ctx context.Context, c *Config, opt *Option, keepImages Images) error { - log.Println("[info] finding expired images") - // find candidates to delete - candidates, err := app.scanRepositories(ctx, c.Repositories, keepImages, opt) - if err != nil { - return err - } if !opt.Delete { return nil } - for name, ids := range candidates { - if err := app.DeleteImages(ctx, name, ids, opt); err != nil { - return err + for _, name := range candidates.RepositoryNames() { + if err := app.DeleteImages(ctx, name, candidates[name], opt.Force); err != nil { + return fmt.Errorf("failed to delete images: %w", err) } } - return nil } -// collectImages collects images in use by ECS tasks / task definitions -func (app *App) collectImages(ctx context.Context, taskdefs []taskdef) (Images, error) { - images := make(Images) - dup := newSet() - for _, td := range taskdefs { - tds := td.String() - if dup.contains(tds) { - continue - } - dup.add(tds) - - ids, err := app.extractECRImages(ctx, tds) - if err != nil { - return nil, err - } - for _, id := range ids { - if images.Add(id, tds) { - log.Printf("[info] image %s is in use by taskdef %s", id.String(), tds) - } - } +func ShowScanResult(s *Scanner, opt *Option) error { + w, err := opt.OutputWriter() + if err != nil { + return fmt.Errorf("failed to open output: %w", err) } - return images, nil -} - -func (app *App) repositories(ctx context.Context) ([]ecrTypes.Repository, error) { - repos := make([]ecrTypes.Repository, 0) - p := ecr.NewDescribeRepositoriesPaginator(app.ecr, &ecr.DescribeRepositoriesInput{}) - for p.HasMorePages() { - repo, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - repos = append(repos, repo.Repositories...) + defer w.Close() + if err := s.Save(w); err != nil { + return fmt.Errorf("failed to save scanned image URIs: %w", err) } - return repos, nil + return nil } -type deletableImageIDs map[RepositoryName][]ecrTypes.ImageIdentifier - -// scanRepositories scans repositories and find expired images -// keepImages is a set of images in use by ECS tasks / task definitions / lambda functions -// so that they are not deleted -func (app *App) scanRepositories(ctx context.Context, rcs []*RepositoryConfig, keepImages Images, opt *Option) (deletableImageIDs, error) { - idsMaps := make(deletableImageIDs) - sums := SummaryTable{} - in := &ecr.DescribeRepositoriesInput{} - if opt.Repository != "" { - in.RepositoryNames = []string{opt.Repository} - } - p := ecr.NewDescribeRepositoriesPaginator(app.ecr, in) - for p.HasMorePages() { - repos, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - REPO: - for _, repo := range repos.Repositories { - name := RepositoryName(*repo.RepositoryName) - var rc *RepositoryConfig - for _, _rc := range rcs { - if _rc.MatchName(name) { - rc = _rc - break - } - } - if rc == nil { - continue REPO - } - imageIDs, sum, err := app.unusedImageIdentifiers(ctx, name, rc, keepImages) - if err != nil { - return nil, err - } - sums = append(sums, sum...) - idsMaps[name] = imageIDs - } - } - sort.SliceStable(sums, func(i, j int) bool { - return sums[i].Repo < sums[j].Repo - }) - log.Printf("[info] output summary to %s as %s", opt.OutputFile, opt.Format) +func ShowSummary(s SummaryTable, opt *Option) error { w, err := opt.OutputWriter() if err != nil { - return nil, fmt.Errorf("failed to open output: %w", err) + return fmt.Errorf("failed to open output: %w", err) } defer w.Close() - if err := sums.print(w, opt.Format); err != nil { - return nil, err - } - return idsMaps, nil + return s.Print(w, opt.Format) } const batchDeleteImageIdsLimit = 100 const batchGetImageLimit = 100 // DeleteImages deletes images from the repository -func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, opt *Option) error { +func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecrTypes.ImageIdentifier, force bool) error { if len(ids) == 0 { log.Println("[info] no need to delete images on", repo) return nil } - if !opt.Delete { - log.Printf("[notice] Expired %d image(s) found on %s. Run delete command to delete them.", len(ids), repo) - return nil - } - if !opt.Force { + if !force { if !prompter.YN(fmt.Sprintf("Do you delete %d images on %s?", len(ids), repo), false) { return errors.New("aborted") } @@ -336,182 +142,9 @@ func (app *App) DeleteImages(ctx context.Context, repo RepositoryName, ids []ecr return nil } -// unusedImageIdentifiers finds image identifiers(by image digests) from the repository. -func (app *App) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, keepImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) { - sums := NewRepoSummary(repo) - images, imageIndexes, sociIndexes, idByTags, err := app.listImageDetails(ctx, repo) - if err != nil { - return nil, sums, err - } - log.Printf("[info] %s has %d images, %d image indexes, %d soci indexes", repo, len(images), len(imageIndexes), len(sociIndexes)) - expiredIds := make([]ecrTypes.ImageIdentifier, 0) - expiredImageIndexes := newSet() - var keepCount int64 -IMAGE: - for _, d := range images { - tag, tagged := imageTag(d) - displayName := string(repo) + ":" + tag - sums.Add(d) - - // Check if the image is in use (digest) - imageURISha256 := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s@%s", *d.RegistryId, app.region, *d.RepositoryName, *d.ImageDigest)) - log.Printf("[debug] checking %s", imageURISha256) - if keepImages.Contains(imageURISha256) { - log.Printf("[info] %s@%s is in used, keep it", repo, *d.ImageDigest) - continue IMAGE - } - - // Check if the image is in use or conditions (tag) - for _, tag := range d.ImageTags { - if rc.MatchTag(tag) { - log.Printf("[info] image %s:%s is matched by tag condition, keep it", repo, tag) - continue IMAGE - } - imageURI := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s", *d.RegistryId, app.region, *d.RepositoryName, tag)) - log.Printf("[debug] checking %s", imageURI) - if keepImages.Contains(imageURI) { - log.Printf("[info] image %s:%s is in used, keep it", repo, tag) - continue IMAGE - } - } - - // Check if the image is expired - pushedAt := *d.ImagePushedAt - if !rc.IsExpired(pushedAt) { - log.Printf("[info] image %s is not expired, keep it", displayName) - continue IMAGE - } - - if tagged { - keepCount++ - if keepCount <= rc.KeepCount { - log.Printf("[info] image %s is in keep_count %d <= %d, keep it", displayName, keepCount, rc.KeepCount) - continue IMAGE - } - } - - // Don't match any conditions, so expired - log.Printf("[notice] image %s is expired %s %s", displayName, *d.ImageDigest, pushedAt.Format(time.RFC3339)) - expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) - sums.Expire(d) - - tagSha256 := strings.Replace(*d.ImageDigest, "sha256:", "sha256-", 1) - if _, found := idByTags[tagSha256]; found { - expiredImageIndexes.add(tagSha256) - } - } - -IMAGE_INDEX: - for _, d := range imageIndexes { - log.Printf("[debug] is an image index %s", *d.ImageDigest) - sums.Add(d) - for _, tag := range d.ImageTags { - if expiredImageIndexes.contains(tag) { - log.Printf("[notice] %s:%s is expired (image index)", repo, tag) - sums.Expire(d) - expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) - continue IMAGE_INDEX - } - } - } - - sociIds, err := app.findSociIndex(ctx, repo, expiredImageIndexes.members()) - if err != nil { - return nil, sums, err - } - -SOCI_INDEX: - for _, d := range sociIndexes { - log.Printf("[debug] is soci index %s", *d.ImageDigest) - sums.Add(d) - for _, id := range sociIds { - if aws.ToString(id.ImageDigest) == aws.ToString(d.ImageDigest) { - log.Printf("[notice] %s@%s is expired (soci index)", repo, *d.ImageDigest) - sums.Expire(d) - expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) - continue SOCI_INDEX - } - } - } - - return expiredIds, sums, nil -} - -func (app *App) listImageDetails(ctx context.Context, repo RepositoryName) ([]ecrTypes.ImageDetail, []ecrTypes.ImageDetail, []ecrTypes.ImageDetail, map[string]ecrTypes.ImageIdentifier, error) { - var images, imageIndexes, sociIndexes []ecrTypes.ImageDetail - foundTags := make(map[string]ecrTypes.ImageIdentifier, 0) - - p := ecr.NewDescribeImagesPaginator(app.ecr, &ecr.DescribeImagesInput{ - RepositoryName: aws.String(string(repo)), - }) - for p.HasMorePages() { - imgs, err := p.NextPage(ctx) - if err != nil { - return nil, nil, nil, nil, err - } - for _, img := range imgs.ImageDetails { - if isContainerImage(img) { - images = append(images, img) - } else if isImageIndex(img) { - imageIndexes = append(imageIndexes, img) - } else if isSociIndex(img) { - sociIndexes = append(sociIndexes, img) - } - for _, tag := range img.ImageTags { - foundTags[tag] = ecrTypes.ImageIdentifier{ImageDigest: img.ImageDigest} - } - } - } - - sort.SliceStable(images, func(i, j int) bool { - return images[i].ImagePushedAt.After(*images[j].ImagePushedAt) - }) - sort.SliceStable(imageIndexes, func(i, j int) bool { - return imageIndexes[i].ImagePushedAt.After(*imageIndexes[j].ImagePushedAt) - }) - sort.SliceStable(sociIndexes, func(i, j int) bool { - return sociIndexes[i].ImagePushedAt.After(*sociIndexes[j].ImagePushedAt) - }) - return images, imageIndexes, sociIndexes, foundTags, nil -} - -func (app *App) findSociIndex(ctx context.Context, repo RepositoryName, imageTags []string) ([]ecrTypes.ImageIdentifier, error) { - ids := make([]ecrTypes.ImageIdentifier, 0, len(imageTags)) - - for _, c := range lo.Chunk(imageTags, batchGetImageLimit) { - imageIds := make([]ecrTypes.ImageIdentifier, 0, len(c)) - for _, tag := range c { - imageIds = append(imageIds, ecrTypes.ImageIdentifier{ImageTag: aws.String(tag)}) - } - res, err := app.ecr.BatchGetImage(ctx, &ecr.BatchGetImageInput{ - ImageIds: imageIds, - RepositoryName: aws.String(string(repo)), - AcceptedMediaTypes: []string{ - string(ociTypes.OCIManifestSchema1), - string(ociTypes.DockerManifestSchema1), - string(ociTypes.DockerManifestSchema2), - }, - }) - if err != nil { - return nil, err - } - for _, img := range res.Images { - if img.ImageManifest == nil { - continue - } - var m oci.IndexManifest - if err := json.Unmarshal([]byte(*img.ImageManifest), &m); err != nil { - log.Printf("[warn] failed to parse manifest: %s %s", *img.ImageManifest, err) - continue - } - for _, d := range m.Manifests { - if d.ArtifactType == MediaTypeSociIndex { - ids = append(ids, ecrTypes.ImageIdentifier{ImageDigest: aws.String(d.Digest.String())}) - } - } - } - } - return ids, nil +func (app *App) GenerateConfig(ctx context.Context, path string) error { + g := NewGenerator(app.awsCfg) + return g.GenerateConfig(ctx, path) } func imageTag(d ecrTypes.ImageDetail) (string, bool) { @@ -523,208 +156,3 @@ func imageTag(d ecrTypes.ImageDetail) (string, bool) { return untaggedStr, false } } - -// scanClusters scans ECS clusters and returns task definitions and images in use -func (app *App) scanClusters(ctx context.Context, clustersConfigs []*ClusterConfig) ([]taskdef, Images, error) { - tds := make([]taskdef, 0) - images := make(Images) - clusterArns, err := app.clusterArns(ctx) - if err != nil { - return tds, nil, err - } - - for _, a := range clusterArns { - var clusterArn string - for _, cc := range clustersConfigs { - if cc.Match(a) { - clusterArn = a - break - } - } - if clusterArn == "" { - continue - } - - log.Printf("[debug] Checking cluster %s", clusterArn) - if _tds, _imgs, err := app.availableResourcesInCluster(ctx, clusterArn); err != nil { - return tds, nil, err - } else { - tds = append(tds, _tds...) - images.Merge(_imgs) - } - } - return tds, images, nil -} - -// collectTaskdefs collects task definitions by configurations -func (app *App) collectTaskdefs(ctx context.Context, tcs []*TaskdefConfig) ([]taskdef, error) { - tds := make([]taskdef, 0) - families, err := app.taskDefinitionFamilies(ctx) - if err != nil { - return tds, err - } - - for _, family := range families { - var name string - var keepCount int64 - for _, tc := range tcs { - if tc.Match(family) { - name = family - keepCount = tc.KeepCount - break - } - } - if name == "" { - continue - } - log.Printf("[debug] Checking task definitions %s latest %d revisions", name, keepCount) - res, err := app.ecs.ListTaskDefinitions(ctx, &ecs.ListTaskDefinitionsInput{ - FamilyPrefix: &name, - MaxResults: aws.Int32(int32(keepCount)), - Sort: ecsTypes.SortOrderDesc, - }) - if err != nil { - return tds, err - } - for _, tdArn := range res.TaskDefinitionArns { - td, err := parseTaskdefArn(tdArn) - if err != nil { - return tds, err - } - tds = append(tds, td) - } - } - return tds, nil -} - -// extractECRImages extracts images (only in ECR) from the task definition -// returns image URIs -func (app App) extractECRImages(ctx context.Context, tdName string) ([]ImageURI, error) { - images := make([]ImageURI, 0) - out, err := app.ecs.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{ - TaskDefinition: &tdName, - }) - if err != nil { - return nil, err - } - for _, container := range out.TaskDefinition.ContainerDefinitions { - u := ImageURI(*container.Image) - if u.IsECRImage() { - images = append(images, u) - } else { - log.Printf("[debug] Skipping non ECR image %s", u) - } - } - return images, nil -} - -// availableResourcesInCluster scans task definitions and images in use in the cluster -func (app *App) availableResourcesInCluster(ctx context.Context, clusterArn string) ([]taskdef, Images, error) { - clusterName := clusterArnToName(clusterArn) - tdArns := make(set) - images := make(Images) - - log.Printf("[debug] Checking tasks in %s", clusterArn) - tp := ecs.NewListTasksPaginator(app.ecs, &ecs.ListTasksInput{Cluster: &clusterArn}) - for tp.HasMorePages() { - to, err := tp.NextPage(ctx) - if err != nil { - return nil, nil, err - } - if len(to.TaskArns) == 0 { - continue - } - tasks, err := app.ecs.DescribeTasks(ctx, &ecs.DescribeTasksInput{ - Cluster: &clusterArn, - Tasks: to.TaskArns, - }) - if err != nil { - return nil, nil, err - } - for _, task := range tasks.Tasks { - tdArn := aws.ToString(task.TaskDefinitionArn) - td, err := parseTaskdefArn(tdArn) - if err != nil { - return nil, nil, err - } - ts, err := arn.Parse(*task.TaskArn) - if err != nil { - return nil, nil, err - } - if tdArns.add(tdArn) { - log.Printf("[info] taskdef %s is used by %s", td.String(), ts.Resource) - } - for _, c := range task.Containers { - u := ImageURI(aws.ToString(c.Image)) - if !u.IsECRImage() { - continue - } - // ECR image - if u.IsDigestURI() { - if images.Add(u, tdArn) { - log.Printf("[info] image %s is used by %s container on %s", u.String(), *c.Name, ts.Resource) - } - } else { - base := u.Base() - digest := aws.ToString(c.ImageDigest) - u := ImageURI(base + "@" + digest) - if images.Add(u, tdArn) { - log.Printf("[info] image %s is used by %s container on %s", u.String(), *c.Name, ts.Resource) - } - } - } - } - } - - sp := ecs.NewListServicesPaginator(app.ecs, &ecs.ListServicesInput{Cluster: &clusterArn}) - for sp.HasMorePages() { - so, err := sp.NextPage(ctx) - if err != nil { - return nil, nil, err - } - if len(so.ServiceArns) == 0 { - continue - } - svs, err := app.ecs.DescribeServices(ctx, &ecs.DescribeServicesInput{ - Cluster: &clusterArn, - Services: so.ServiceArns, - }) - if err != nil { - return nil, nil, err - } - for _, sv := range svs.Services { - log.Printf("[debug] Checking service %s", *sv.ServiceName) - for _, dp := range sv.Deployments { - tdArn := aws.ToString(dp.TaskDefinition) - td, err := parseTaskdefArn(tdArn) - if err != nil { - return nil, nil, err - } - if tdArns.add(tdArn) { - log.Printf("[info] taskdef %s is used by %s deployment on service %s/%s", td.String(), *dp.Status, *sv.ServiceName, clusterName) - } - } - } - } - var tds []taskdef - for a := range tdArns { - td, err := parseTaskdefArn(a) - if err != nil { - return nil, nil, err - } - tds = append(tds, td) - } - return tds, images, nil -} - -func arnToName(name, removePrefix string) string { - if arn.IsARN(name) { - a, _ := arn.Parse(name) - return strings.Replace(a.Resource, removePrefix, "", 1) - } - return name -} - -func clusterArnToName(arn string) string { - return arnToName(arn, "cluster/") -} diff --git a/generate.go b/generate.go index 64edb17..e7053bc 100644 --- a/generate.go +++ b/generate.go @@ -12,6 +12,10 @@ import ( "strings" "github.com/Songmu/prompter" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ecr" + "github.com/aws/aws-sdk-go-v2/service/ecs" + "github.com/aws/aws-sdk-go-v2/service/lambda" "github.com/goccy/go-yaml" ) @@ -25,18 +29,28 @@ func nameToPattern(s string) string { return s } -func (app *App) GenerateConfig(ctx context.Context, configFile string, opt *Option) error { +type Generator struct { + awsCfg aws.Config +} + +func NewGenerator(cfg aws.Config) *Generator { + return &Generator{ + awsCfg: cfg, + } +} + +func (g *Generator) GenerateConfig(ctx context.Context, configFile string) error { config := Config{} - if err := app.generateClusterConfig(ctx, &config); err != nil { + if err := g.generateClusterConfig(ctx, &config); err != nil { return err } - if err := app.generateTaskdefConfig(ctx, &config); err != nil { + if err := g.generateTaskdefConfig(ctx, &config); err != nil { return err } - if err := app.generateLambdaConfig(ctx, &config); err != nil { + if err := g.generateLambdaConfig(ctx, &config); err != nil { return err } - if err := app.generateRepositoryConfig(ctx, &config); err != nil { + if err := g.generateRepositoryConfig(ctx, &config); err != nil { return err } @@ -59,8 +73,8 @@ func (app *App) GenerateConfig(ctx context.Context, configFile string, opt *Opti return nil } -func (app *App) generateClusterConfig(ctx context.Context, config *Config) error { - clusters, err := app.clusterArns(ctx) +func (g *Generator) generateClusterConfig(ctx context.Context, config *Config) error { + clusters, err := clusterArns(ctx, ecs.NewFromConfig(g.awsCfg)) if err != nil { return err } @@ -89,8 +103,8 @@ func (app *App) generateClusterConfig(ctx context.Context, config *Config) error return nil } -func (app *App) generateTaskdefConfig(ctx context.Context, config *Config) error { - taskdefs, err := app.taskDefinitionFamilies(ctx) +func (g *Generator) generateTaskdefConfig(ctx context.Context, config *Config) error { + taskdefs, err := taskDefinitionFamilies(ctx, ecs.NewFromConfig(g.awsCfg)) if err != nil { return err } @@ -121,8 +135,8 @@ func (app *App) generateTaskdefConfig(ctx context.Context, config *Config) error return nil } -func (app *App) generateLambdaConfig(ctx context.Context, config *Config) error { - lambdas, err := app.lambdaFunctions(ctx) +func (g *Generator) generateLambdaConfig(ctx context.Context, config *Config) error { + lambdas, err := lambdaFunctions(ctx, lambda.NewFromConfig(g.awsCfg)) if err != nil { return err } @@ -154,8 +168,8 @@ func (app *App) generateLambdaConfig(ctx context.Context, config *Config) error return nil } -func (app *App) generateRepositoryConfig(ctx context.Context, config *Config) error { - repos, err := app.repositories(ctx) +func (g *Generator) generateRepositoryConfig(ctx context.Context, config *Config) error { + repos, err := ecrRepositories(ctx, ecr.NewFromConfig(g.awsCfg)) if err != nil { return err } diff --git a/go.mod b/go.mod index b04a525..3ceb969 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/fatih/color v1.17.0 github.com/fujiwara/logutils v1.1.0 github.com/goccy/go-yaml v1.9.4 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.20.0 github.com/k1LoW/duration v1.1.0 github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 07b4b36..2476cd6 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,8 @@ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7a github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8= github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.0 h1:wRqHpOeVh3DnenOrPy9xDOLdnLatiGuuNRVelR2gSbg= github.com/google/go-containerregistry v0.20.0/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= diff --git a/lambda.go b/lambda.go index a315968..fc7fd3d 100644 --- a/lambda.go +++ b/lambda.go @@ -12,31 +12,11 @@ import ( lambdaTypes "github.com/aws/aws-sdk-go-v2/service/lambda/types" ) -func (app *App) lambdaFunctions(ctx context.Context) ([]lambdaTypes.FunctionConfiguration, error) { - fns := make([]lambdaTypes.FunctionConfiguration, 0) - p := lambda.NewListFunctionsPaginator(app.lambda, &lambda.ListFunctionsInput{}) - for p.HasMorePages() { - r, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - for _, fn := range r.Functions { - if fn.PackageType != "Image" { - continue - } - log.Printf("[debug] lambda function %s PackageType %s", *fn.FunctionName, fn.PackageType) - fns = append(fns, fn) - } - } - return fns, nil -} - -func (app *App) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) (Images, error) { - funcs, err := app.lambdaFunctions(ctx) +func (s *Scanner) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) error { + funcs, err := lambdaFunctions(ctx, s.lambda) if err != nil { - return nil, err + return err } - images := make(Images) for _, fn := range funcs { var name string @@ -54,7 +34,7 @@ func (app *App) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) (I } log.Printf("[debug] Checking Lambda function %s latest %d versions", name, keepCount) p := lambda.NewListVersionsByFunctionPaginator( - app.lambda, + s.lambda, &lambda.ListVersionsByFunctionInput{ FunctionName: fn.FunctionName, MaxItems: aws.Int32(int32(keepCount)), @@ -64,7 +44,7 @@ func (app *App) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) (I for p.HasMorePages() { r, err := p.NextPage(ctx) if err != nil { - return nil, err + return err } versions = append(versions, r.Versions...) } @@ -76,23 +56,23 @@ func (app *App) scanLambdaFunctions(ctx context.Context, lcs []*LambdaConfig) (I } for _, v := range versions { log.Println("[debug] Getting Lambda function ", *v.FunctionArn) - f, err := app.lambda.GetFunction(ctx, &lambda.GetFunctionInput{ + f, err := s.lambda.GetFunction(ctx, &lambda.GetFunctionInput{ FunctionName: v.FunctionArn, }) if err != nil { - return nil, err + return err } u := ImageURI(aws.ToString(f.Code.ImageUri)) if u == "" { continue } log.Println("[debug] ImageUri", u) - if images.Add(u, aws.ToString(v.FunctionArn)) { + if s.Images.Add(u, aws.ToString(v.FunctionArn)) { log.Printf("[info] %s is in use by Lambda function %s:%s", u.String(), *v.FunctionName, *v.Version) } } } - return images, nil + return nil } func lambdaVersionInt64(v string) int64 { diff --git a/option.go b/option.go new file mode 100644 index 0000000..9fec84e --- /dev/null +++ b/option.go @@ -0,0 +1,39 @@ +package ecrm + +import ( + "fmt" + "io" + "os" +) + +type Option struct { + ScanOnly bool + Scan bool + Delete bool + Force bool + Repository RepositoryName + OutputFile string + Format outputFormat + ScannedFiles []string +} + +func (opt *Option) Validate() error { + if len(opt.ScannedFiles) == 0 && !opt.Scan { + return fmt.Errorf("no --scanned-files and --no-scan provided. specify at least one") + } + return nil +} + +// NopCloserWriter is a writer that does nothing on Close +type NopCloserWriter struct { + io.Writer +} + +func (NopCloserWriter) Close() error { return nil } + +func (opt *Option) OutputWriter() (io.WriteCloser, error) { + if opt.OutputFile == "" || opt.OutputFile == "-" { + return NopCloserWriter{os.Stdout}, nil + } + return os.Create(opt.OutputFile) +} diff --git a/planner.go b/planner.go new file mode 100644 index 0000000..c3eac9a --- /dev/null +++ b/planner.go @@ -0,0 +1,260 @@ +package ecrm + +import ( + "context" + "encoding/json" + "fmt" + "log" + "sort" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ecr" + ecrTypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" + oci "github.com/google/go-containerregistry/pkg/v1" + ociTypes "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/samber/lo" +) + +type Planner struct { + ecr *ecr.Client + region string +} + +func NewPlanner(cfg aws.Config) *Planner { + return &Planner{ + ecr: ecr.NewFromConfig(cfg), + region: cfg.Region, + } +} + +type DeletableImageIDs map[RepositoryName][]ecrTypes.ImageIdentifier + +func (d DeletableImageIDs) RepositoryNames() []RepositoryName { + names := lo.Keys(d) + sort.Slice(names, func(i, j int) bool { + return names[i] < names[j] + }) + return names +} + +// Plan scans repositories and find expired images, and returns a summary table and a map of deletable image identifiers. +// +// keepImages is a set of images in use by ECS tasks / task definitions / lambda functions +// so that they are not deleted +func (p *Planner) Plan(ctx context.Context, rcs []*RepositoryConfig, keepImages Images, repo RepositoryName) (SummaryTable, DeletableImageIDs, error) { + idsMaps := make(DeletableImageIDs) + sums := SummaryTable{} + in := &ecr.DescribeRepositoriesInput{} + if repo != "" { + in.RepositoryNames = []string{string(repo)} + } + pager := ecr.NewDescribeRepositoriesPaginator(p.ecr, in) + for pager.HasMorePages() { + repos, err := pager.NextPage(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to describe repositories: %w", err) + } + REPO: + for _, repo := range repos.Repositories { + name := RepositoryName(*repo.RepositoryName) + var rc *RepositoryConfig + for _, _rc := range rcs { + if _rc.MatchName(name) { + rc = _rc + break + } + } + if rc == nil { + continue REPO + } + imageIDs, sum, err := p.unusedImageIdentifiers(ctx, name, rc, keepImages) + if err != nil { + return nil, nil, fmt.Errorf("failed to find unused image identifiers: %w", err) + } + sums = append(sums, sum...) + idsMaps[name] = imageIDs + } + } + sums.Sort() + return sums, idsMaps, nil +} + +// unusedImageIdentifiers finds image identifiers(by image digests) from the repository. +func (p *Planner) unusedImageIdentifiers(ctx context.Context, repo RepositoryName, rc *RepositoryConfig, keepImages Images) ([]ecrTypes.ImageIdentifier, RepoSummary, error) { + sums := NewRepoSummary(repo) + images, imageIndexes, sociIndexes, idByTags, err := p.listImageDetails(ctx, repo) + if err != nil { + return nil, sums, err + } + log.Printf("[info] %s has %d images, %d image indexes, %d soci indexes", repo, len(images), len(imageIndexes), len(sociIndexes)) + expiredIds := make([]ecrTypes.ImageIdentifier, 0) + expiredImageIndexes := newSet() + var keepCount int64 +IMAGE: + for _, d := range images { + tag, tagged := imageTag(d) + displayName := string(repo) + ":" + tag + sums.Add(d) + + // Check if the image is in use (digest) + imageURISha256 := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s@%s", *d.RegistryId, p.region, *d.RepositoryName, *d.ImageDigest)) + log.Printf("[debug] checking %s", imageURISha256) + if keepImages.Contains(imageURISha256) { + log.Printf("[info] %s@%s is in used, keep it", repo, *d.ImageDigest) + continue IMAGE + } + + // Check if the image is in use or conditions (tag) + for _, tag := range d.ImageTags { + if rc.MatchTag(tag) { + log.Printf("[info] image %s:%s is matched by tag condition, keep it", repo, tag) + continue IMAGE + } + imageURI := ImageURI(fmt.Sprintf("%s.dkr.ecr.%s.amazonaws.com/%s:%s", *d.RegistryId, p.region, *d.RepositoryName, tag)) + log.Printf("[debug] checking %s", imageURI) + if keepImages.Contains(imageURI) { + log.Printf("[info] image %s:%s is in used, keep it", repo, tag) + continue IMAGE + } + } + + // Check if the image is expired + pushedAt := *d.ImagePushedAt + if !rc.IsExpired(pushedAt) { + log.Printf("[info] image %s is not expired, keep it", displayName) + continue IMAGE + } + + if tagged { + keepCount++ + if keepCount <= rc.KeepCount { + log.Printf("[info] image %s is in keep_count %d <= %d, keep it", displayName, keepCount, rc.KeepCount) + continue IMAGE + } + } + + // Don't match any conditions, so expired + log.Printf("[notice] image %s is expired %s %s", displayName, *d.ImageDigest, pushedAt.Format(time.RFC3339)) + expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) + sums.Expire(d) + + tagSha256 := strings.Replace(*d.ImageDigest, "sha256:", "sha256-", 1) + if _, found := idByTags[tagSha256]; found { + expiredImageIndexes.add(tagSha256) + } + } + +IMAGE_INDEX: + for _, d := range imageIndexes { + log.Printf("[debug] is an image index %s", *d.ImageDigest) + sums.Add(d) + for _, tag := range d.ImageTags { + if expiredImageIndexes.contains(tag) { + log.Printf("[notice] %s:%s is expired (image index)", repo, tag) + sums.Expire(d) + expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) + continue IMAGE_INDEX + } + } + } + + sociIds, err := p.findSociIndex(ctx, repo, expiredImageIndexes.members()) + if err != nil { + return nil, sums, fmt.Errorf("failed to find soci index: %w", err) + } + +SOCI_INDEX: + for _, d := range sociIndexes { + log.Printf("[debug] is soci index %s", *d.ImageDigest) + sums.Add(d) + for _, id := range sociIds { + if aws.ToString(id.ImageDigest) == aws.ToString(d.ImageDigest) { + log.Printf("[notice] %s@%s is expired (soci index)", repo, *d.ImageDigest) + sums.Expire(d) + expiredIds = append(expiredIds, ecrTypes.ImageIdentifier{ImageDigest: d.ImageDigest}) + continue SOCI_INDEX + } + } + } + + return expiredIds, sums, nil +} + +func (p *Planner) listImageDetails(ctx context.Context, repo RepositoryName) ([]ecrTypes.ImageDetail, []ecrTypes.ImageDetail, []ecrTypes.ImageDetail, map[string]ecrTypes.ImageIdentifier, error) { + var images, imageIndexes, sociIndexes []ecrTypes.ImageDetail + foundTags := make(map[string]ecrTypes.ImageIdentifier, 0) + + pager := ecr.NewDescribeImagesPaginator(p.ecr, &ecr.DescribeImagesInput{ + RepositoryName: aws.String(string(repo)), + }) + for pager.HasMorePages() { + imgs, err := pager.NextPage(ctx) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed to describe images: %w", err) + } + for _, img := range imgs.ImageDetails { + if isContainerImage(img) { + images = append(images, img) + } else if isImageIndex(img) { + imageIndexes = append(imageIndexes, img) + } else if isSociIndex(img) { + sociIndexes = append(sociIndexes, img) + } + for _, tag := range img.ImageTags { + foundTags[tag] = ecrTypes.ImageIdentifier{ImageDigest: img.ImageDigest} + } + } + } + + sort.SliceStable(images, func(i, j int) bool { + return images[i].ImagePushedAt.After(*images[j].ImagePushedAt) + }) + sort.SliceStable(imageIndexes, func(i, j int) bool { + return imageIndexes[i].ImagePushedAt.After(*imageIndexes[j].ImagePushedAt) + }) + sort.SliceStable(sociIndexes, func(i, j int) bool { + return sociIndexes[i].ImagePushedAt.After(*sociIndexes[j].ImagePushedAt) + }) + return images, imageIndexes, sociIndexes, foundTags, nil +} + +func (p *Planner) findSociIndex(ctx context.Context, repo RepositoryName, imageTags []string) ([]ecrTypes.ImageIdentifier, error) { + ids := make([]ecrTypes.ImageIdentifier, 0, len(imageTags)) + + for _, c := range lo.Chunk(imageTags, batchGetImageLimit) { + imageIds := make([]ecrTypes.ImageIdentifier, 0, len(c)) + for _, tag := range c { + imageIds = append(imageIds, ecrTypes.ImageIdentifier{ImageTag: aws.String(tag)}) + } + res, err := p.ecr.BatchGetImage(ctx, &ecr.BatchGetImageInput{ + ImageIds: imageIds, + RepositoryName: aws.String(string(repo)), + AcceptedMediaTypes: []string{ + string(ociTypes.OCIManifestSchema1), + string(ociTypes.DockerManifestSchema1), + string(ociTypes.DockerManifestSchema2), + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to batch get image: %w", err) + } + for _, img := range res.Images { + if img.ImageManifest == nil { + continue + } + var m oci.IndexManifest + if err := json.Unmarshal([]byte(*img.ImageManifest), &m); err != nil { + log.Printf("[warn] failed to parse manifest: %s %s", *img.ImageManifest, err) + continue + } + for _, d := range m.Manifests { + if d.ArtifactType == MediaTypeSociIndex { + ids = append(ids, ecrTypes.ImageIdentifier{ImageDigest: aws.String(d.Digest.String())}) + } + } + } + } + return ids, nil +} diff --git a/scanner.go b/scanner.go new file mode 100644 index 0000000..bcf2783 --- /dev/null +++ b/scanner.go @@ -0,0 +1,290 @@ +package ecrm + +import ( + "context" + "io" + "log" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/service/ecs" + ecsTypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" + "github.com/aws/aws-sdk-go-v2/service/lambda" +) + +type Scanner struct { + Images Images + + ecs *ecs.Client + lambda *lambda.Client +} + +func NewScanner(cfg aws.Config) *Scanner { + return &Scanner{ + Images: make(Images), + ecs: ecs.NewFromConfig(cfg), + lambda: lambda.NewFromConfig(cfg), + } +} + +func (s *Scanner) Scan(ctx context.Context, c *Config) error { + log.Println("[info] scanning resources") + + // collect images in use by ECS tasks / task definitions + var taskdefs []taskdef + if tds, err := s.scanClusters(ctx, c.Clusters); err != nil { + return err + } else { + taskdefs = append(taskdefs, tds...) + } + if tds, err := s.collectTaskdefs(ctx, c.TaskDefinitions); err != nil { + return err + } else { + taskdefs = append(taskdefs, tds...) + } + if err := s.collectImages(ctx, taskdefs); err != nil { + return err + } + + // collect images in use by lambda functions + if err := s.scanLambdaFunctions(ctx, c.LambdaFunctions); err != nil { + return err + } + + return nil +} + +func (s *Scanner) LoadFiles(files []string) error { + for _, f := range files { + log.Println("[info] loading scanned image URIs from", f) + imgs := make(Images) + if err := imgs.LoadFile(f); err != nil { + return err + } + log.Println("[info] loaded", len(imgs), "image URIs") + s.Images.Merge(imgs) + } + return nil +} + +func (s *Scanner) Save(w io.Writer) error { + log.Println("[info] saving scanned image URIs") + if err := s.Images.Print(w); err != nil { + return err + } + log.Println("[info] saved", len(s.Images), "image URIs") + return nil +} + +// collectImages collects images in use by ECS tasks / task definitions +func (s *Scanner) collectImages(ctx context.Context, taskdefs []taskdef) error { + dup := newSet() + for _, td := range taskdefs { + tds := td.String() + if dup.contains(tds) { + continue + } + dup.add(tds) + + ids, err := s.extractECRImages(ctx, tds) + if err != nil { + return err + } + for _, id := range ids { + if s.Images.Add(id, tds) { + log.Printf("[info] image %s is in use by taskdef %s", id.String(), tds) + } + } + } + return nil +} + +// extractECRImages extracts images (only in ECR) from the task definition +// returns image URIs +func (s *Scanner) extractECRImages(ctx context.Context, tdName string) ([]ImageURI, error) { + images := make([]ImageURI, 0) + out, err := s.ecs.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{ + TaskDefinition: &tdName, + }) + if err != nil { + return nil, err + } + for _, container := range out.TaskDefinition.ContainerDefinitions { + u := ImageURI(*container.Image) + if u.IsECRImage() { + images = append(images, u) + } else { + log.Printf("[debug] Skipping non ECR image %s", u) + } + } + return images, nil +} + +// scanClusters scans ECS clusters and returns task definitions and images in use +func (s *Scanner) scanClusters(ctx context.Context, clustersConfigs []*ClusterConfig) ([]taskdef, error) { + tds := make([]taskdef, 0) + clusterArns, err := clusterArns(ctx, s.ecs) + if err != nil { + return nil, err + } + + for _, a := range clusterArns { + var clusterArn string + for _, cc := range clustersConfigs { + if cc.Match(a) { + clusterArn = a + break + } + } + if clusterArn == "" { + continue + } + + log.Printf("[debug] Checking cluster %s", clusterArn) + if _tds, err := s.availableResourcesInCluster(ctx, clusterArn); err != nil { + return tds, err + } else { + tds = append(tds, _tds...) + } + } + return tds, nil +} + +// collectTaskdefs collects task definitions by configurations +func (s *Scanner) collectTaskdefs(ctx context.Context, tcs []*TaskdefConfig) ([]taskdef, error) { + tds := make([]taskdef, 0) + families, err := taskDefinitionFamilies(ctx, s.ecs) + if err != nil { + return tds, err + } + + for _, family := range families { + var name string + var keepCount int64 + for _, tc := range tcs { + if tc.Match(family) { + name = family + keepCount = tc.KeepCount + break + } + } + if name == "" { + continue + } + log.Printf("[debug] Checking task definitions %s latest %d revisions", name, keepCount) + res, err := s.ecs.ListTaskDefinitions(ctx, &ecs.ListTaskDefinitionsInput{ + FamilyPrefix: &name, + MaxResults: aws.Int32(int32(keepCount)), + Sort: ecsTypes.SortOrderDesc, + }) + if err != nil { + return tds, err + } + for _, tdArn := range res.TaskDefinitionArns { + td, err := parseTaskdefArn(tdArn) + if err != nil { + return tds, err + } + tds = append(tds, td) + } + } + return tds, nil +} + +// availableResourcesInCluster scans task definitions and images in use in the cluster +func (s *Scanner) availableResourcesInCluster(ctx context.Context, clusterArn string) ([]taskdef, error) { + clusterName := clusterArnToName(clusterArn) + tdArns := make(set) + + log.Printf("[debug] Checking tasks in %s", clusterArn) + tp := ecs.NewListTasksPaginator(s.ecs, &ecs.ListTasksInput{Cluster: &clusterArn}) + for tp.HasMorePages() { + to, err := tp.NextPage(ctx) + if err != nil { + return nil, err + } + if len(to.TaskArns) == 0 { + continue + } + tasks, err := s.ecs.DescribeTasks(ctx, &ecs.DescribeTasksInput{ + Cluster: &clusterArn, + Tasks: to.TaskArns, + }) + if err != nil { + return nil, err + } + for _, task := range tasks.Tasks { + tdArn := aws.ToString(task.TaskDefinitionArn) + td, err := parseTaskdefArn(tdArn) + if err != nil { + return nil, err + } + ts, err := arn.Parse(*task.TaskArn) + if err != nil { + return nil, err + } + if tdArns.add(tdArn) { + log.Printf("[info] taskdef %s is used by %s", td.String(), ts.Resource) + } + for _, c := range task.Containers { + u := ImageURI(aws.ToString(c.Image)) + if !u.IsECRImage() { + continue + } + // ECR image + if u.IsDigestURI() { + if s.Images.Add(u, tdArn) { + log.Printf("[info] image %s is used by %s container on %s", u.String(), *c.Name, ts.Resource) + } + } else { + base := u.Base() + digest := aws.ToString(c.ImageDigest) + u := ImageURI(base + "@" + digest) + if s.Images.Add(u, tdArn) { + log.Printf("[info] image %s is used by %s container on %s", u.String(), *c.Name, ts.Resource) + } + } + } + } + } + + sp := ecs.NewListServicesPaginator(s.ecs, &ecs.ListServicesInput{Cluster: &clusterArn}) + for sp.HasMorePages() { + so, err := sp.NextPage(ctx) + if err != nil { + return nil, err + } + if len(so.ServiceArns) == 0 { + continue + } + svs, err := s.ecs.DescribeServices(ctx, &ecs.DescribeServicesInput{ + Cluster: &clusterArn, + Services: so.ServiceArns, + }) + if err != nil { + return nil, err + } + for _, sv := range svs.Services { + log.Printf("[debug] Checking service %s", *sv.ServiceName) + for _, dp := range sv.Deployments { + tdArn := aws.ToString(dp.TaskDefinition) + td, err := parseTaskdefArn(tdArn) + if err != nil { + return nil, err + } + if tdArns.add(tdArn) { + log.Printf("[info] taskdef %s is used by %s deployment on service %s/%s", td.String(), *dp.Status, *sv.ServiceName, clusterName) + } + } + } + } + var tds []taskdef + for a := range tdArns { + td, err := parseTaskdefArn(a) + if err != nil { + return nil, err + } + tds = append(tds, td) + } + return tds, nil +} diff --git a/summary.go b/summary.go index 96c7728..041389c 100644 --- a/summary.go +++ b/summary.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "sort" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -120,7 +121,13 @@ const ( type SummaryTable []*Summary -func (s *SummaryTable) print(w io.Writer, format outputFormat) error { +func (s SummaryTable) Sort() { + sort.SliceStable(s, func(i, j int) bool { + return s[i].Repo < s[j].Repo + }) +} + +func (s *SummaryTable) Print(w io.Writer, format outputFormat) error { switch format { case formatTable: return s.printTable(w) diff --git a/util.go b/util.go index b71f624..97cee0b 100644 --- a/util.go +++ b/util.go @@ -1,8 +1,17 @@ package ecrm import ( + "context" + "log" + "strings" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/service/ecr" ecrTypes "github.com/aws/aws-sdk-go-v2/service/ecr/types" + "github.com/aws/aws-sdk-go-v2/service/ecs" + "github.com/aws/aws-sdk-go-v2/service/lambda" + lambdaTypes "github.com/aws/aws-sdk-go-v2/service/lambda/types" ociTypes "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -27,3 +36,74 @@ func isImageIndex(d ecrTypes.ImageDetail) bool { func isSociIndex(d ecrTypes.ImageDetail) bool { return ociTypes.MediaType(aws.ToString(d.ArtifactMediaType)) == MediaTypeSociIndex } + +func taskDefinitionFamilies(ctx context.Context, client *ecs.Client) ([]string, error) { + tds := make([]string, 0) + p := ecs.NewListTaskDefinitionFamiliesPaginator(client, &ecs.ListTaskDefinitionFamiliesInput{}) + for p.HasMorePages() { + td, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + log.Println("[debug] task definition families:", td.Families) + tds = append(tds, td.Families...) + } + return tds, nil +} + +func clusterArns(ctx context.Context, client *ecs.Client) ([]string, error) { + clusters := make([]string, 0) + p := ecs.NewListClustersPaginator(client, &ecs.ListClustersInput{}) + for p.HasMorePages() { + co, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + clusters = append(clusters, co.ClusterArns...) + } + return clusters, nil +} + +func lambdaFunctions(ctx context.Context, client *lambda.Client) ([]lambdaTypes.FunctionConfiguration, error) { + fns := make([]lambdaTypes.FunctionConfiguration, 0) + p := lambda.NewListFunctionsPaginator(client, &lambda.ListFunctionsInput{}) + for p.HasMorePages() { + r, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + for _, fn := range r.Functions { + if fn.PackageType != "Image" { + continue + } + log.Printf("[debug] lambda function %s PackageType %s", *fn.FunctionName, fn.PackageType) + fns = append(fns, fn) + } + } + return fns, nil +} + +func ecrRepositories(ctx context.Context, client *ecr.Client) ([]ecrTypes.Repository, error) { + repos := make([]ecrTypes.Repository, 0) + p := ecr.NewDescribeRepositoriesPaginator(client, &ecr.DescribeRepositoriesInput{}) + for p.HasMorePages() { + repo, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + repos = append(repos, repo.Repositories...) + } + return repos, nil +} + +func arnToName(name, removePrefix string) string { + if arn.IsARN(name) { + a, _ := arn.Parse(name) + return strings.Replace(a.Resource, removePrefix, "", 1) + } + return name +} + +func clusterArnToName(arn string) string { + return arnToName(arn, "cluster/") +}