build-notebooks-TEMPLATE.yaml

# inspired by
# https://github.com/thesuperzapper/kubeflow/blob/master/.github/workflows/example_notebook_servers_publish_TEMPLATE.yaml
---
name: Build & Publish Notebook Servers (TEMPLATE)
"on":
  workflow_call:
    inputs:
      # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
      # https://docs.github.com/en/actions/learn-github-actions/contexts
      target:
        required: true
        description: "make target to build"
        type: string
      github:
        required: true
        description: "top workflow's `github`"
        type: string

jobs:
  build:
    strategy:
      matrix:
        os: [ubuntu-22.04]
    runs-on: ${{matrix.os}}
    env:
      # Some pieces of code (image pulls for example) in podman consult TMPDIR or default to /var/tmp
      TMPDIR: /home/runner/.local/share/containers/tmpdir
      # Use the rootful instance of podman for sharing images with cri-o
      # https://podman-desktop.io/blog/sharing-podman-images-with-kubernetes-cluster#introduction
      # https://access.redhat.com/solutions/6986565
      CONTAINER_HOST: unix:///var/run/podman/podman.sock
      # We don't push here when building PRs, so we can use the same IMAGE_REGISTRY in all branches of the workflow
      IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
      # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
      CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
      TRIVY_VERSION: 0.57.1
      TRIVY_VULNDB: "/home/runner/.local/share/containers/trivy_db"
      # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
      TRIVY_SCAN_FS_JSON: '{}'
      # Poetry version for use in running tests
      POETRY_VERSION: '2.0.0'

    steps:

      - uses: actions/checkout@v4

      - run: mkdir -p $TMPDIR

      # for bin/buildinputs in scripts/sandbox.py
      - uses: actions/setup-go@v5
        with:
          cache-dependency-path: "**/*.sum"

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # region Free up disk space

      - name: Free up additional disk space
        # https://docs.github.com/en/actions/learn-github-actions/expressions
        if: "${{ contains(inputs.target, 'rocm') || contains(inputs.target, 'cuda') ||
         contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
        run: |
          set -x

          df -h

          sudo apt-get update
          sudo apt-get remove -y '^dotnet-.*'
          sudo apt-get remove -y '^llvm-.*'
          sudo apt-get remove -y 'php.*'
          sudo apt-get remove -y '^mongodb-.*'
          sudo apt-get autoremove -y
          sudo apt-get clean
          sudo rm -rf /usr/local/.ghcup &
          sudo rm -rf /usr/local/lib/android &
          sudo rm -rf /usr/local/share/boost &
          sudo rm -rf /usr/local/lib/node_modules &
          sudo rm -rf /usr/share/dotnet &
          sudo rm -rf /opt/ghc &
          sudo rm -rf /opt/hostedtoolcache/CodeQL &

          sudo docker image prune --all --force &

          wait

          df -h

      - name: Mount lvm overlay for podman builds
        run: |
          df -h
          free -h

          bash ./ci/cached-builds/gha_lvm_overlay.sh

          df -h
          free -h

      # endregion

      # region Podman setup

      # https://github.com/containers/buildah/issues/2521#issuecomment-884779112
      - name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
        run: sudo apt-get -qq remove podman crun

      - uses: actions/cache@v4
        id: cached-linuxbrew
        with:
          path: /home/linuxbrew/.linuxbrew
          key: linuxbrew

      - name: Install podman
        if: steps.cached-linuxbrew.outputs.cache-hit != 'true'
        run: |
          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
          /home/linuxbrew/.linuxbrew/bin/brew install podman

      - name: Add linuxbrew to PATH
        run: echo "/home/linuxbrew/.linuxbrew/bin/" >> $GITHUB_PATH

      - name: Configure Podman
        run: |
          set -Eeuxo pipefail

          # podman running as service ignores the TMPDIR env var here, let's give it a bind-mount to /var/tmp
          mkdir -p $TMPDIR
          sudo mount --bind -o rw,noexec,nosuid,nodev,bind $TMPDIR /var/tmp

          # podman from brew has its own /etc (was giving me Failed to obtain podman configuration: runroot must be set)
          # the (default) config location is also where cri-o gets its storage defaults (that can be overriden in crio.conf)
          sudo cp ci/cached-builds/containers.conf /etc/containers.conf
          sudo cp ci/cached-builds/containers.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers.conf
          sudo cp ci/cached-builds/storage.conf /etc/containers/storage.conf
          sudo cp ci/cached-builds/storage.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/storage.conf
          sudo cp ci/cached-builds/registries.conf /etc/containers/registries.conf
          sudo cp ci/cached-builds/registries.conf /home/linuxbrew/.linuxbrew/opt/podman/etc/containers/registries.conf

          # should reset storage when changing storage.conf
          mkdir -p $HOME/.local/share/containers/storage/tmp
          # remote (CONTAINER_HOST) podman does not do reset (and refuses --force option)
          sudo /home/linuxbrew/.linuxbrew/opt/podman/bin/podman system reset --force

          # https://github.com/containers/podman/blob/main/docs/tutorials/socket_activation.md
          # since `brew services start podman` is buggy, let's do our own brew-compatible service
          # Regarding directory paths, see https://unix.stackexchange.com/questions/224992/where-do-i-put-my-systemd-unit-file
          sudo mkdir -p /usr/local/lib/systemd/system/
          sudo cp ci/cached-builds/podman.service /usr/local/lib/systemd/system/podman.service
          sudo cp ci/cached-builds/podman.socket /usr/local/lib/systemd/system/podman.socket
          sudo systemctl daemon-reload
          sudo systemctl unmask --now podman.service podman.socket
          sudo systemctl start podman.socket

          # needed (much) later for trivy
          echo "PODMAN_SOCK=/var/run/podman/podman.sock" >> $GITHUB_ENV

          # quick check podman works
          podman ps

      - name: Show error logs (on failure)
        if: ${{ failure() }}
        run: journalctl -xe

      - name: Calculate image name and tag
        id: calculated_vars
        run: |
          SANITIZED_REF_NAME=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g')
          IMAGE_TAG="${SANITIZED_REF_NAME}_${{ github.sha }}"

          echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
          echo "OUTPUT_IMAGE=${{ env.IMAGE_REGISTRY}}:${{ inputs.target }}-${IMAGE_TAG}" >> "$GITHUB_OUTPUT"

      # endregion

      # region Trivy init & DB pre-pull

      - name: "pull_request|schedule: resolve target if Trivy scan should run"
        id: resolve-target
        if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
        env:
          EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
          HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
          FS_SCAN_FOLDER: ${{ fromJson(env.TRIVY_SCAN_FS_JSON)[inputs.target] }}
        run: |
          if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
            if [[ -n "$FS_SCAN_FOLDER" ]]; then
              TARGET="$FS_SCAN_FOLDER"
              TYPE="fs"
            else
              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
              TYPE="image"
            fi
          elif [[ "$EVENT_NAME" == "schedule" ]]; then
            if [[ -n "$FS_SCAN_FOLDER" ]]; then
              TARGET="$FS_SCAN_FOLDER"
              TYPE="fs"
            else
              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
              TYPE="image"
            fi
          fi

          if [[ -n "$TARGET" ]]; then
            echo "target=$TARGET" >> $GITHUB_OUTPUT
            echo "type=$TYPE" >> $GITHUB_OUTPUT
            echo "Trivy scan will run on $TARGET ($TYPE)"
          else
            echo "Trivy scan won't run"
          fi

      # only one db can be downloaded in one call https://github.com/aquasecurity/trivy/issues/3616
      - name: Pre-pull Trivy vulnerabilities DB
        if: ${{ steps.resolve-target.outputs.target }}
        run: |
          mkdir ${TRIVY_VULNDB}
          podman run --rm \
            --env PODMAN_SOCK \
            -v ${TRIVY_VULNDB}:/cache \
            docker.io/aquasec/trivy:$TRIVY_VERSION \
              --cache-dir /cache \
              image \
              --download-db-only
          podman run --rm \
            --env PODMAN_SOCK \
            -v ${TRIVY_VULNDB}:/cache \
            docker.io/aquasec/trivy:$TRIVY_VERSION \
              --cache-dir /cache \
              image \
              --download-java-db-only

      # endregion

      # region Image build

      # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
      - name: "push|schedule: make ${{ inputs.target }}"
        run: |
          make ${{ inputs.target }}
        if: ${{ fromJson(inputs.github).event_name == 'push' || fromJson(inputs.github).event_name == 'schedule' }}
        env:
          IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
          CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
          # dependent images were already built and pushed, so just let podman pull it
          BUILD_DEPENDENT_IMAGES: "no"

      # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
      - name: "pull_request: make ${{ inputs.target }}"
        run: |
          make ${{ inputs.target }}
        if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}"
        env:
          IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
          CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"
          # We don't have access to image registry, so disable pushing
          PUSH_IMAGES: "no"

      - name: "Show podman images information"
        run: podman images --digests

      # endregion

      # region Pytest image tests

      - name: Install poetry
        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
        run: pipx install poetry==${{ env.POETRY_VERSION }}
        env:
          PIPX_HOME: /home/runner/.local/pipx
          PIPX_BIN_DIR: /home/runner/.local/bin

      - name: Check poetry is installed correctly
        run: poetry env info

      - name: Set up Python
        id: setup-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: 'poetry'

      - name: Configure poetry
        run: poetry env use "${{ steps.setup-python.outputs.python-path }}"

      - name: Install deps
        run: poetry install --sync

      - name: Run container tests (in PyTest)
        run: |
          set -Eeuxo pipefail
          # retry to increase CI reliability
          for i in {1..5}; do
            if podman pull --retry 10 "${RYUK_CONTAINER_IMAGE}"; then break; fi
          done
          # now run the tests
          poetry run pytest --capture=fd tests/containers --image="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
        env:
          DOCKER_HOST: "unix:///var/run/podman/podman.sock"
          TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/podman/podman.sock"
          RYUK_CONTAINER_IMAGE: "testcontainers/ryuk:0.8.1"

      # endregion Pytest image tests

      # region Makefile image tests

      - name: "Check if we have tests or not"
        id: have-tests
        run: "ci/cached-builds/has_tests.py --target ${{ inputs.target }}"

      - name: "Change pull policy to IfNotPresent"
        run: |
          set -Eeuxo pipefail

          find . \( -name "statefulset.yaml" -o -name "pod.yaml" \) -type f -exec \
            sed -i'' 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/g' {} \;
          git diff

      # [INFO] Running command (('make deploy9-runtimes-rocm-tensorflow-ubi9-python-3.11',), {'shell': True})
      # Deploying notebook from runtimes/rocm/tensorflow/ubi9-python-3.11/kustomize/base directory...
      # sed: can't read runtimes/rocm/tensorflow/ubi9-python-3.11/kustomize/base/kustomization.yaml: No such file or directory
      - name: "Fixup paths that prevent us from running rocm tests"
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail

          mkdir -p runtimes/rocm
          ln -s ../rocm-tensorflow runtimes/rocm/tensorflow
          ln -s ../rocm-pytorch runtimes/rocm/pytorch

      # https://cri-o.io/
      - name: Install cri-o
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail

          sudo apt-get update
          sudo apt-get install -y software-properties-common curl

          curl -fsSL https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/Release.key | \
            sudo gpg --dearmor --batch --yes -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$KUBERNETES_VERSION/deb/ /" | \
            sudo tee /etc/apt/sources.list.d/kubernetes.list

          curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/Release.key | \
            sudo gpg --dearmor --batch --yes -o /etc/apt/keyrings/cri-o-apt-keyring.gpg

          echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/$CRIO_VERSION/deb/ /" | \
            sudo tee /etc/apt/sources.list.d/cri-o.list

          sudo apt-get update
          sudo apt-get install -y cri-o kubelet kubeadm kubectl

          # make use of /etc/cni/net.d/11-crio-ipv4-bridge.conflist so we don't
          # need a pod network and just use the default bridge
          sudo rm -rf /etc/cni/net.d/*
          # cat /etc/cni/net.d/11-crio-ipv4-bridge.conflist
          # https://github.com/containerd/containerd/blob/main/script%2Fsetup%2Finstall-cni
          # https://www.cni.dev/plugins/current/main/bridge/
          sudo cp ci/cached-builds/11-crio-ipv4-bridge.conflist /etc/cni/net.d/11-crio-ipv4-bridge.conflist

          sudo cp ci/cached-builds/crio.conf /etc/crio/crio.conf.d/

          sudo systemctl start crio.service
        env:
          CRIO_VERSION: v1.30
          KUBERNETES_VERSION: v1.30

      - name: Show crio debug data (on failure)
        if: ${{ failure() && steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail

          sudo systemctl status crio.service || true
          sudo journalctl -xeu crio.service

      # do this early, it's a good check that cri-o is not completely broken
      - name: "Show crio images information"
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: sudo crictl images

      - name: Install Kubernetes cluster
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail

          sudo swapoff -a
          sudo modprobe br_netfilter
          sudo sysctl -w net.ipv4.ip_forward=1

          # Was getting strange DNS resolution errors from pods that don't seem to want to go away sometimes:
          #  Resolving raw.githubusercontent.com (raw.githubusercontent.com)... failed: Name or service not known.
          #  wget: unable to resolve host address ‘raw.githubusercontent.com’
          # Here's what helped:
          #  https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/#known-issues
          #  https://github.com/kubernetes/kubernetes/blob/e4c1f980b76fecece30c2f77885a7117192170a6/CHANGELOG/CHANGELOG-1.30.md?plain=1#L1454
          #  https://github.com/canonical/microk8s/issues/68#issuecomment-404923563
          sudo ufw allow in on cni0
          sudo ufw allow out on cni0
          sudo ufw default allow routed
          sudo iptables -P FORWARD ACCEPT
          sudo iptables -t nat -A POSTROUTING -s 10.85.0.0/16 -o eth0 -j MASQUERADE

          # https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm
          sudo kubeadm init --config=ci/cached-builds/kubeadm.yaml

          mkdir -p $HOME/.kube
          sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
          sudo chown $(id -u):$(id -g) $HOME/.kube/config

      - name: Show kubelet debug data (on failure)
        if: ${{ failure() && steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail

          sudo systemctl status kubelet || true
          sudo journalctl -xeu kubelet

          # Here is one example how you may list all running Kubernetes containers by using crictl:
          sudo crictl --runtime-endpoint unix:///var/run/crio/crio.sock ps -a | grep kube | grep -v pause
          # Once you have found the failing container, you can inspect its logs with:
          # crictl --runtime-endpoint unix:///var/run/crio/crio.sock logs CONTAINERID

      - name: Untaint the master
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: kubectl taint nodes --all node-role.kubernetes.io/control-plane-

      - name: Show nodes status and wait for readiness
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: |
          kubectl describe nodes
          kubectl wait --for=condition=Ready nodes --all --timeout=100s || (kubectl describe nodes && false)

      - name: Wait for pods to be running
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: |
          set -Eeuxo pipefail
          kubectl wait deployments --all --all-namespaces --for=condition=Available --timeout=100s
          kubectl wait pods --all --all-namespaces --for=condition=Ready --timeout=100s

      - name: "Run image tests"
        if: ${{ steps.have-tests.outputs.tests == 'true' }}
        run: python3 ci/cached-builds/make_test.py --target ${{ inputs.target }}
        env:
          IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
          # for make deploy, mandatory to specify for the more exotic cases
          NOTEBOOK_TAG: "${{ inputs.target }}-${{ steps.calculated_vars.outputs.IMAGE_TAG }}"

      # endregion

      # region Trivy vulnerability scan

      - name: Run Trivy vulnerability scanner
        if: ${{ steps.resolve-target.outputs.target }}
        run: |
          REPORT_FOLDER=${{ github.workspace }}/report
          REPORT_FILE=trivy-report.md
          REPORT_TEMPLATE=trivy-markdown.tpl

          mkdir -p $REPORT_FOLDER
          cp ci/$REPORT_TEMPLATE $REPORT_FOLDER

          SCAN_TARGET=${{ steps.resolve-target.outputs.target }}
          SCAN_TYPE=${{ steps.resolve-target.outputs.type }}
          echo "Scanning $SCAN_TARGET ($SCAN_TYPE)"

          if [[ "$SCAN_TYPE" == "image" ]]; then
            SCAN_ARGS="--image-src podman --podman-host /var/run/podman/podman.sock"
            PODMAN_ARGS="-v ${PODMAN_SOCK}:/var/run/podman/podman.sock"
          elif [[ "$SCAN_TYPE" == "fs" ]]; then
            WORKSPACE_FOLDER="/workspace"
            SCAN_TARGET="$WORKSPACE_FOLDER/$SCAN_TARGET"
            PODMAN_ARGS="-v ${{ github.workspace }}:$WORKSPACE_FOLDER"
          fi

          # have trivy access podman socket,
          # https://github.com/aquasecurity/trivy/issues/580#issuecomment-666423279
          podman run --rm \
              $PODMAN_ARGS \
              -v ${REPORT_FOLDER}:/report \
              -v ${TRIVY_VULNDB}:/cache \
              docker.io/aquasec/trivy:$TRIVY_VERSION \
                --cache-dir /cache \
                $SCAN_TYPE \
                $SCAN_ARGS \
                --skip-db-update \
                --scanners vuln --ignore-unfixed \
                --exit-code 0 --timeout 30m \
                --format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
                $SCAN_TARGET

          cat $REPORT_FOLDER/$REPORT_FILE >> $GITHUB_STEP_SUMMARY

      # endregion

      # region Typescript (browser) image tests

      # https://playwright.dev/docs/ci
      # https://playwright.dev/docs/docker
      # we leave little free disk space after we mount LVM for podman storage
      # not enough to install playwright; running playwright in podman uses the space we have
      - name: Run Playwright tests
        if: ${{ contains(inputs.target, 'codeserver') }}
        # --ipc=host because Microsoft says so in Playwright docs
        # --net=host because testcontainers connects to the Reaper container's exposed port
        # we need to pass through the relevant environment variables
        #  DEBUG configures nodejs debuggers, sets different verbosity as needed
        #  CI=true is set on every CI nowadays
        #  PODMAN_SOCK should be mounted to /var/run/docker.sock, other likely mounting locations may not exist (mkdir -p)
        #  TEST_TARGET is the workbench image the test will run
        # --volume(s) let us access docker socket and not clobber host's node_modules
        run: |
          podman run \
            --interactive --rm \
            --ipc=host \
            --net=host \
            --env "CI=true" \
            --env "NPM_CONFIG_fund=false" \
            --env "DEBUG=testcontainers:*" \
            --env "PODMAN_SOCK=/var/run/docker.sock" \
            --env "TEST_TARGET" \
            --volume ${PODMAN_SOCK}:/var/run/docker.sock \
            --volume ${PWD}:/mnt \
            --volume /mnt/node_modules \
            mcr.microsoft.com/playwright:v1.48.1-noble \
            /bin/bash <<EOF
              set -Eeuxo pipefail
              cd /mnt
              npm install -g pnpm && pnpm install
              pnpm exec playwright test
              exit 0
          EOF
        working-directory: tests/browser
        env:
          TEST_TARGET: "${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
      - uses: actions/upload-artifact@v4
        if: ${{ !cancelled() && fromJson(inputs.github).event_name == 'pull_request' && contains(inputs.target, 'codeserver') }}
        with:
          name: "${{ inputs.target }}_playwright-report"
          path: tests/browser/playwright-report/
          retention-days: 30

      # endregion

      - run: df -h
        if: "${{ !cancelled() }}"