From a22afa9f3f03d519b72b96af56897ceebf195787 Mon Sep 17 00:00:00 2001 From: You-Sheng Yang Date: Fri, 7 Jun 2024 22:44:25 +0800 Subject: [PATCH] Build multiarch images Signed-off-by: You-Sheng Yang --- .github/workflows/ci.yml | 307 +++++++++++++++++++++++++++++++++++---- 1 file changed, 276 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d04fac..8ac98756 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,17 @@ on: branches: - master workflow_dispatch: + inputs: + wanted: + description: > + Build selected suites. Use "all" for all buildable suites, "active" + for all non-EOL suites, "eol" for all EOL-ed suites, or + comma-separated codenames. Default is "active". + type: string + default: 'active' schedule: - cron: 0 0 * * 0 -defaults: - run: - shell: 'bash -Eeuo pipefail -x {0}' - concurrency: group: ${{ github.ref }} cancel-in-progress: true @@ -28,36 +32,277 @@ jobs: name: Generate Jobs runs-on: ubuntu-latest outputs: - strategy: ${{ steps.generate-jobs.outputs.strategy }} + codenames: ${{ steps.processing.outputs.codenames }} steps: - - uses: actions/checkout@v4 - - uses: docker-library/bashbrew@HEAD - - id: generate-jobs - name: Generate Jobs + - name: Debian Releases Info + id: debian + uses: vicamo/actions-library/debian-releases@v1 + - name: Ubuntu Releases Info + id: ubuntu + uses: vicamo/actions-library/ubuntu-releases@v1 + + - name: Post Processing + id: processing + env: + DEBIAN_JSON: ${{ steps.debian.outputs.json }} + UBUNTU_JSON: ${{ steps.ubuntu.outputs.json }} run: | - strategy="$("$BASHBREW_SCRIPTS/github-actions/generate.sh")" + platforms='[ + ["amd64", "linux/amd64"], + ["arm", "linux/arm"], + ["arm64", "linux/arm64"], + ["armel", "linux/arm/v5"], + ["armhf", "linux/arm/v7"], + ["i386", "linux/386"], + ["loong64", "linux/loong64"], + ["mips64el", "linux/mips64le"], + ["ppc64el", "linux/ppc64le"], + ["riscv64", "linux/riscv64"], + ["s390x", "linux/s390x"], + + ["mips", "linux/mips"], + ["mipsel", "linux/mipsle"], + ["powerpc", "linux/ppc"], + ["ppc64", "linux/ppc64"], + ["s390", "linux/s390"], + ["sparc", "linux/sparc"], + ["sparc64", "linux/sparc64"], + ["x32", "linux/amd64p32"], + + ["alpha", "linux/alpha"], + ["hppa", "linux/hppa"], + ["m68k", "linux/m68k"], + ["sh4", "linux/sh4"] + ]' + disabled_codename='["experimental"]' + disabled_arches='[ + "hurd-amd64", + "hurd-i386", + "ia64", + "kfreebsd-amd64", + "kfreebsd-i386", + "s390", + "x32" + ]' + codenames="$({ echo "${DEBIAN_JSON}"; echo "${UBUNTU_JSON}"; } | + jq -s 'map(.[])' | + jq -c -M 'map(select(.codename as $c | + '"${disabled_codename}"' | + index($c) == null)) | + map(. as $s | + $s.mirrors[0].pockets[$s.codename].architectures as $da | + $s | + { + "distribution":.distribution, + "codename":.codename, + "suite":.suite, + "active":.active, + "architectures":(.architectures - '"${disabled_arches}"' | + map({ + "arch":., + "platform":(. as $arch | + '"${platforms}"' | + map(select(.[0] == $arch))[0][1]), + }) | + tostring) + })')" - EOF="EOF-$RANDOM-$RANDOM-$RANDOM" - echo "strategy<<$EOF" >> "$GITHUB_OUTPUT" - jq <<<"$strategy" . | tee -a "$GITHUB_OUTPUT" - echo "$EOF" >> "$GITHUB_OUTPUT" + case "${{ inputs.wanted || 'active' }}" in + all) ;; + active) + codenames="$(echo "${codenames}" | jq -c -M 'map(select(.active))')" + ;; + eol) + codenames="$(echo "${codenames}" | jq -c -M 'map(select(.active | not))')" + ;; + *) + wanted="$(echo "${{ inputs.wanted }}" | sed 's/ //g; s/\([^,]\+\)/"\1"/g')" + codenames="$(echo "${codenames}" | + jq -c -M 'map(. as $row | + ['"${wanted}"'] | .[] | select($row.codename == .) | + $row)')" + ;; + esac - test: + echo "::group::Built JSON(codenames)" + echo "${codenames}" | jq + echo "::endgroup::" + + echo "codenames=${codenames}" | tee -a "${GITHUB_OUTPUT}" + + build: needs: generate-jobs - strategy: ${{ fromJson(needs.generate-jobs.outputs.strategy) }} - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: ${{ fromJson(needs.generate-jobs.outputs.codenames) }} + name: Build + runs-on: ubuntu-latest + env: + DISTRO: ${{ matrix.distribution }} + CODENAME: ${{ matrix.codename }} + SUITE: ${{ matrix.suite }} + REPOSITORY: 'vicamo/buildpack-deps' + DRY_RUN: ${{ github.ref_name != 'main' }} steps: - - uses: actions/checkout@v4 - - name: Prepare Environment - run: ${{ matrix.runs.prepare }} - - name: Pull Dependencies - run: ${{ matrix.runs.pull }} - - name: Build ${{ matrix.name }} - run: ${{ matrix.runs.build }} - - name: History ${{ matrix.name }} - run: ${{ matrix.runs.history }} - - name: Test ${{ matrix.name }} - run: ${{ matrix.runs.test }} - - name: '"docker images"' - run: ${{ matrix.runs.images }} + + - name: Set up QEMU (docker/setup-qemu-action) + uses: docker/setup-qemu-action@v3 + with: + image: tonistiigi/binfmt:qemu-v9.2.0 + + - name: Checkout + uses: actions/checkout@v4 + + - name: Apply templates + run: | + if [ ! -e "${DISTRO}/${CODENAME}/Dockerfile" ]; then + ./versions.sh "${DISTRO}/${CODENAME}" + ./apply-templates.sh "${DISTRO}/${CODENAME}" + fi + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + android: true + docker-images: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Setup containerd image store + run: | + echo "::group::docker daemon config" + { cat /etc/docker/daemon.json || echo '{}'; } | \ + jq '. | .+{"features": {"containerd-snapshotter": true, "buildkit": true}}' | \ + sudo tee /etc/docker/daemon.json.new + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + echo "::endgroup::" + + sudo systemctl restart docker + + echo "::group::docker driver status" + docker info -f '{{ .DriverStatus }}' + echo "::endgroup::" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + # While we're running docker daemon with containerd-snapshotter, the + # built containers exported in 'load' output from build-and-push + # action are not visible to a following build session using a + # non-default builder. Don't switch to the newly created build here to + # work-around this issue. + # + # See https://github.com/moby/buildkit/issues/2343 + use: false + version: v0.20.1 + + - name: Login to Docker Hub + if: ${{ github.ref_name == 'main' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Prepare + id: prepare + run: | + set -x + builder_platforms_json="[$(docker buildx inspect|grep ^Platforms:|cut -d: -f2|tr -d \*|sed 's/\([^ ,]\+\)/"\1"/g')]" + architectures_json="$(echo '${{ matrix.architectures }}' | + jq -c -M 'map(select(.platform as $c | + '"${builder_platforms_json}"' | + index($c) != null) + )')" + platforms="$(echo "${architectures_json}" | jq -r 'map(.platform)|join(",")')" + + echo "platforms=${platforms}" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta for curl images + id: meta-curl + uses: docker/metadata-action@v5 + with: + images: ${{ env.REPOSITORY }} + tags: | + ${{ matrix.codename }}-curl + ${{ matrix.suite && format('{0}-curl', matrix.suite) || '' }} + ${{ matrix.suite == 'stable' && 'latest-curl' || '' }} + flavor: | + latest=false + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + - name: Build curl images + id: build-curl + uses: docker/build-push-action@v5 + with: + annotations: ${{ contains(steps.prepare.outputs.platforms, ',') && steps.meta-curl.outputs.annotations || '' }} + build-args: | + BASEIMAGE=${{ format('vicamo/{0}:{1}', matrix.distribution, matrix.codename) }} + context: ${{ format('{0}/{1}/curl', matrix.distribution, matrix.codename) }} + labels: ${{ steps.meta-curl.outputs.labels }} + load: true + platforms: ${{ steps.prepare.outputs.platforms }} + provenance: false + pull: false + tags: ${{ steps.meta-curl.outputs.tags }} + + - name: Docker meta for scm images + id: meta-scm + uses: docker/metadata-action@v5 + with: + images: ${{ env.REPOSITORY }} + tags: | + ${{ matrix.codename }}-scm + ${{ matrix.suite && format('{0}-scm', matrix.suite) || '' }} + ${{ matrix.suite == 'stable' && 'latest-scm' || '' }} + flavor: | + latest=false + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + - name: Build scm images + id: build-scm + uses: docker/build-push-action@v5 + with: + annotations: ${{ contains(steps.prepare.outputs.platforms, ',') && steps.meta-scm.outputs.annotations || '' }} + build-args: | + BASEIMAGE=${{ format('{0}:{1}-curl', env.REPOSITORY, matrix.codename) }} + context: ${{ format('{0}/{1}/scm', matrix.distribution, matrix.codename) }} + labels: ${{ steps.meta-scm.outputs.labels }} + load: true + platforms: ${{ steps.prepare.outputs.platforms }} + provenance: false + pull: false + tags: ${{ steps.meta-scm.outputs.tags }} + + - name: Docker meta for buildpack-deps images + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REPOSITORY }} + tags: | + ${{ matrix.codename }} + ${{ matrix.suite && format('{0}', matrix.suite) || '' }} + flavor: | + latest=${{ matrix.suite == 'stable' && 'true' || 'false' }} + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + - name: Build buildpack-deps images + id: build + uses: docker/build-push-action@v5 + with: + annotations: ${{ contains(steps.prepare.outputs.platforms, ',') && steps.meta.outputs.annotations || '' }} + build-args: | + BASEIMAGE=${{ format('{0}:{1}-scm', env.REPOSITORY, matrix.codename) }} + context: ${{ format('{0}/{1}', matrix.distribution, matrix.codename) }} + labels: ${{ steps.meta.outputs.labels }} + load: true + platforms: ${{ steps.prepare.outputs.platforms }} + provenance: false + pull: false + tags: ${{ steps.meta.outputs.tags }}