Build multi-arch snapshot containers (#29419)

* Build multi-arch snapshot containers * Add push-containers * Setup buildx * Remove push arg * Broaden check * Missing ) * Push with docker task * Test out SOURCE_DATE_EPOCH * Back to TagPush * Save containers instead of pushing * Try containerImageTags * Test multiple tags * Update docker plugin * Fix * Fix * Turn off provenance * Fix jenkins precommits * Fix jenkins precommits * Remove source_date_epoch * Don't delete containers if they have dependencies that cant be deleted
apache · Nov 17, 2023 · 5e9ea74 · 5e9ea74
1 parent 06265f9
commit 5e9ea74
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 5 deletions.
diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
@@ -78,6 +78,8 @@ jobs:
           comment_phrase: ${{ matrix.job_phrase }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           github_job: ${{ matrix.job_name }} (${{ matrix.container_task }})
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Authenticate on GCP
         uses: google-github-actions/setup-gcloud@v0
         with:
@@ -101,8 +103,10 @@ jobs:
       - name: run Publish Beam SDK Snapshots script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:${{ matrix.container_task }}:dockerTagPush
+          gradle-command: :sdks:${{ matrix.container_task }}:docker
           arguments: |
             -Pjava11Home=$JAVA_HOME_11_X64 \
             -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \
-            -Pdocker-tag-list=${{ github.sha }},latest
+            -Pdocker-tag-list=${{ github.sha }},latest \
+            -Pcontainer-architecture-list=arm64,amd64 \
+            -Ppush-containers \
diff --git a/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh b/.test-infra/tools/stale_dataflow_prebuilt_image_cleaner.sh
@@ -70,8 +70,31 @@ for image_name in ${IMAGE_NAMES[@]}; do
       # do not delete the one with latest label and the newest image without latest label
       # this make sure we leave at least one container under each image name, either labelled "latest" or not
       if [ "$LATEST_IN_TIME" != "$current" ]; then
-        echo "Deleting image. Command: gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q"
-        gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q || FAILED_TO_DELETE+="${current} "
+        # Check to see if this image is built on top of earlier images. This is the case for multiarch images,
+        # they will have a virtual size of 0 and a created date at the start of the epoch, but their manifests will
+        # point to active images. These images should only be deleted when all of their dependencies can be safely
+        # deleted.
+        MANIFEST=$(docker manifest inspect ${image_name}@"${current}")
+        SHOULD_DELETE=0
+        DIGEST=$(echo $MANIFEST |  jq -r '.manifests[0].digest')
+        if [ "$DIGEST" != "null" ]
+        then
+          SHOULD_DELETE=1
+          for i in ${STALE_IMAGES_CURRENT[@]}
+          do
+            echo "$i"
+            if [ "$i" = "$DIGEST" ]
+            then
+              SHOULD_DELETE=0
+            fi
+          done
+        fi
+
+        if [ $SHOULD_DELETE = 0 ]
+        then
+          echo "Deleting image. Command: gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q"
+          gcloud container images delete ${image_name}@"${current}" --force-delete-tags -q || FAILED_TO_DELETE+="${current} "
+        fi
       fi
     done
   fi

diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamDockerPlugin.groovy
@@ -240,6 +240,11 @@ class BeamDockerPlugin implements Plugin<Project> {
       if (ext.builder != null) {
         buildCommandLine.addAll('--builder', ext.builder)
       }
+      // Jenkins dependencies aren't up to date enough to accept this flag.
+      // Temporarily exclude until we fully move to GHA.
+      if (!ext.project.jenkins.isCIBuild) {
+        buildCommandLine.addAll('--provenance=false')
+      }
     } else {
       buildCommandLine.add 'build'
     }
@@ -267,7 +272,19 @@ class BeamDockerPlugin implements Plugin<Project> {
     if (ext.pull) {
       buildCommandLine.add '--pull'
     }
-    buildCommandLine.addAll(['-t', "${-> ext.name}", '.'])
+    if (!ext.tags.isEmpty() && ext.push) {
+      String[] repoParts = (ext.name as String).split(':')
+      String repo = repoParts[0]
+      for (int i = 1; i < repoParts.length - 1; i++) {
+        repo += ':' + repoParts[i]
+      }
+      for (tag in ext.getTags()) {
+        buildCommandLine.addAll(['-t', repo + ':' + tag])
+      }
+      buildCommandLine.add '.'
+    } else {
+      buildCommandLine.addAll(['-t', "${-> ext.name}", '.'])
+    }
     logger.debug("${buildCommandLine}" as String)
     return buildCommandLine
   }