fix: clean up setup scripts to support ARM64 (autowarefoundation#219)

* fix: clean up setup scripts to support ARM64

Signed-off-by: Kenji Miyake <[email protected]>

* fix push condition

Signed-off-by: Kenji Miyake <[email protected]>

* remove comment lines

Signed-off-by: Kenji Miyake <[email protected]>

* rename env names

Signed-off-by: Kenji Miyake <[email protected]>

* use context values

Signed-off-by: Kenji Miyake <[email protected]>

* fix mistake

Signed-off-by: Kenji Miyake <[email protected]>

* copy env files to Docker image

Signed-off-by: Kenji Miyake <[email protected]>

* fix typo

Signed-off-by: Kenji Miyake <[email protected]>

* fix bug

Signed-off-by: Kenji Miyake <[email protected]>

* define Docker args after FROM

Signed-off-by: Kenji Miyake <[email protected]>

.github/actions/docker-build-and-push/action.yaml

@@ -5,28 +5,16 @@ inputs:
   bake-target:
     description: ""
     required: true
-  platforms:
+  build-args:
     description: ""
-    required: true
+    required: false
   tag-suffix:
     description: ""
-    required: true
+    required: false
 
 runs:
   using: composite
   steps:
-    - name: Get branch name
-      id: get-branch-name
-      run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
-      shell: bash
-
-    - name: Check branch of workflow_dispatch
-      if: ${{ github.event_name == 'workflow_dispatch' && steps.get-branch-name.outputs.branch != github.event.repository.default_branch }}
-      run: |
-        echo "workflow_dispatch is allowed only with the branch '${{ github.event.repository.default_branch }}', '${{ steps.get-branch-name.outputs.branch }}' is not allowed."
-        exit 1
-      shell: bash
-
     - name: Setup Docker Buildx
       uses: docker/setup-buildx-action@v1
 
@@ -104,9 +92,9 @@ runs:
     - name: Build and push
       uses: docker/bake-action@v1
       with:
-        push: ${{ github.event_name != 'pull_request' }}
+        push: ${{ github.ref_name == github.event.repository.default_branch }}
         files: |
           docker/${{ inputs.bake-target }}/docker-bake.hcl
           bake.json
         set: |
-          *.platform=${{ inputs.platforms }}
+          ${{ inputs.build-args }}

.github/workflows/docker-build-and-push-self-hosted.yaml

@@ -23,11 +23,23 @@ jobs:
       - name: Free disk space
         uses: ./.github/actions/free-disk-space
 
+      - name: Load env
+        run: |
+          cat amd64.env | sed "/^#/d" >> $GITHUB_ENV
+          if [ "$(uname -m)" = "aarch64" ]; then
+            cat arm64.env | sed "/^#/d" >> $GITHUB_ENV
+          fi
+
       - name: Build 'autoware-universe'
         uses: ./.github/actions/docker-build-and-push
         with:
           bake-target: autoware-universe
-          platforms: linux/arm64
+          build-args: |
+            *.platform=linux/arm64
+            *.args.ROS_DISTRO=${{ env.rosdistro }}
+            *.args.CUDA_IMAGE_TAG=${{ env.cuda_image_tag }}
+            *.args.CUDNN_VERSION=${{ env.cudnn_version }}
+            *.args.TENSORRT_VERSION=${{ env.tensorrt_version }}
           tag-suffix: -arm64
 
       - name: Show disk space

.github/workflows/docker-build-and-push.yaml

@@ -18,11 +18,23 @@ jobs:
       - name: Free disk space
         uses: ./.github/actions/free-disk-space
 
+      - name: Load env
+        run: |
+          cat amd64.env | sed "/^#/d" >> $GITHUB_ENV
+          if [ "$(uname -m)" = "aarch64" ]; then
+            cat arm64.env | sed "/^#/d" >> $GITHUB_ENV
+          fi
+
       - name: Build 'autoware-universe'
         uses: ./.github/actions/docker-build-and-push
         with:
           bake-target: autoware-universe
-          platforms: linux/amd64
+          build-args: |
+            *.platform=linux/amd64
+            *.args.ROS_DISTRO=${{ env.rosdistro }}
+            *.args.CUDA_IMAGE_TAG=${{ env.cuda_image_tag }}
+            *.args.CUDNN_VERSION=${{ env.cudnn_version }}
+            *.args.TENSORRT_VERSION=${{ env.tensorrt_version }}
           tag-suffix: -amd64
 
       - name: Show disk space

.shellcheckrc

@@ -0,0 +1 @@
+external-sources=true

amd64.env

@@ -0,0 +1,6 @@
+rosdistro=galactic
+rmw_implementation=rmw_cyclonedds_cpp
+cuda_version=11.4
+cuda_image_tag=11.4.1-devel-ubuntu20.04
+cudnn_version=8.2.4.15-1+cuda11.4
+tensorrt_version=8.2.4-1+cuda11.4

ansible/playbooks/core.yaml

@@ -1,8 +1,11 @@
 - hosts: localhost
   connection: local
-  vars:
-    - rosdistro: galactic
-    - rmw_implementation: rmw_cyclonedds_cpp
+  pre_tasks:
+    - name: Print args
+      ansible.builtin.debug:
+        msg:
+          - rosdistro: "{{ rosdistro }}"
+          - rmw_implementation: "{{ rmw_implementation }}"
   roles:
     - role: autoware.dev_env.autoware_core
     - role: autoware.dev_env.docker

ansible/playbooks/universe.yaml

@@ -1,18 +1,21 @@
 - hosts: localhost
   connection: local
-  vars:
-    - rosdistro: galactic
-    - rmw_implementation: rmw_cyclonedds_cpp
-    - cuda_version: 11-4
-    - cudnn_version: 8.2.4.15-1+cuda11.4
-    - tensorrt_version: 8.2.4-1+cuda11.4
   vars_prompt:
     - name: install_nvidia
       prompt: |-
         [Warning] Some components depend on NVIDIA libraries (CUDA, cuDNN, TensorRT), which require license agreements.
         Did you confirm the licenses and are you sure to to install them? [y/N]
       private: false
   pre_tasks:
+    - name: Print args
+      ansible.builtin.debug:
+        msg:
+          - rosdistro: "{{ rosdistro }}"
+          - rmw_implementation: "{{ rmw_implementation }}"
+          - cuda_version: "{{ cuda_version }}"
+          - cudnn_version: "{{ cudnn_version }}"
+          - tensorrt_version: "{{ tensorrt_version }}"
+
     - name: Show warnings that NVIDIA libraries will not be installed
       ansible.builtin.pause:
         seconds: 10

ansible/roles/cuda/tasks/main.yaml

@@ -25,10 +25,15 @@
     repo: deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/{{ cuda_architecture.stdout }}/ /
     state: present
 
-- name: Install cuda-{{ cuda_version }}
+- name: Get dash-case name of cuda_version
+  ansible.builtin.shell: bash -c 'sed -e "s/\./-/g" <<< $(echo {{ cuda_version }})'
+  register: dash_case_cuda_version
+  changed_when: false
+
+- name: Install cuda-{{ dash_case_cuda_version.stdout }}
   become: true
   ansible.builtin.apt:
-    name: cuda-{{ cuda_version }}
+    name: cuda-{{ dash_case_cuda_version.stdout }}
     update_cache: true
 
 - name: Add PATH to .bashrc

ansible/roles/rmw_implementation/tasks/main.yaml

@@ -1,12 +1,12 @@
-- name: Get kebab-case name
+- name: Get dash-case name of rmw_implementation
   ansible.builtin.shell: bash -c 'sed -e "s/_/-/g" <<< $(echo {{ rmw_implementation }})'
-  register: kebab_case_name
+  register: dash_case_rmw_implementation
   changed_when: false
 
-- name: Install ros-{{ rosdistro }}-{{ kebab_case_name.stdout }}
+- name: Install ros-{{ rosdistro }}-{{ dash_case_rmw_implementation.stdout }}
   become: true
   ansible.builtin.apt:
-    name: ros-{{ rosdistro }}-{{ kebab_case_name.stdout }}
+    name: ros-{{ rosdistro }}-{{ dash_case_rmw_implementation.stdout }}
     state: latest
     update_cache: true
 

arm64.env

@@ -0,0 +1,2 @@
+# Override amd64's settings
+cudnn_version=8.2.4.12-1+cuda11.4

docker/autoware-universe/Dockerfile

@@ -1,6 +1,11 @@
-FROM nvidia/cuda:11.4.1-devel-ubuntu20.04 as devel
+ARG CUDA_IMAGE_TAG
+FROM nvidia/cuda:$CUDA_IMAGE_TAG as devel
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+ARG ROS_DISTRO
+ARG CUDNN_VERSION
+ARG TENSORRT_VERSION
+
 ## Install apt packages
 # hadolint ignore=DL3008
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
@@ -10,7 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-ins
   && rm -rf /var/lib/apt/lists/*
 
 ## Copy files
-COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml /autoware/
+COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/
 COPY ansible/ /autoware/ansible/
 WORKDIR /autoware
 RUN ls /autoware
@@ -25,38 +30,36 @@ RUN --mount=type=ssh \
   && mkdir src \
   && vcs import src < autoware.repos \
   && rosdep update \
-  && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro galactic \
+  && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
 ## Install cuDNN and TensorRT
-ARG CUDNN_VERSION=8.2.4.15-1+cuda11.4
-ARG TENSORRT_VERSION=8.2.4-1+cuda11.4
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
-  libcudnn8=$CUDNN_VERSION \
-  libcudnn8-dev=$CUDNN_VERSION \
-  libnvinfer8=$TENSORRT_VERSION \
-  libnvinfer-dev=$TENSORRT_VERSION \
-  libnvinfer-plugin8=$TENSORRT_VERSION \
-  libnvinfer-plugin-dev=$TENSORRT_VERSION \
-  libnvonnxparsers8=$TENSORRT_VERSION \
-  libnvonnxparsers-dev=$TENSORRT_VERSION \
-  libnvparsers8=$TENSORRT_VERSION \
-  libnvparsers-dev=$TENSORRT_VERSION \
+  libcudnn8="$CUDNN_VERSION" \
+  libcudnn8-dev="$CUDNN_VERSION" \
+  libnvinfer8="$TENSORRT_VERSION" \
+  libnvinfer-dev="$TENSORRT_VERSION" \
+  libnvinfer-plugin8="$TENSORRT_VERSION" \
+  libnvinfer-plugin-dev="$TENSORRT_VERSION" \
+  libnvonnxparsers8="$TENSORRT_VERSION" \
+  libnvonnxparsers-dev="$TENSORRT_VERSION" \
+  libnvparsers8="$TENSORRT_VERSION" \
+  libnvparsers-dev="$TENSORRT_VERSION" \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
 ## Create entrypoint
 # hadolint ignore=SC1091
-RUN echo "source /opt/ros/galactic/setup.bash" > /etc/bash.bashrc
+RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc
 CMD ["/bin/bash"]
 
 FROM devel as prebuilt
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 ## Build and change permission for runtime data conversion
 # hadolint ignore=SC1091
-RUN source /opt/ros/galactic/setup.bash \
+RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \
   && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \
   && find /autoware/install -type d -exec chmod 777 {} \;
 

docker/build.sh

@@ -5,11 +5,21 @@ set -e
 SCRIPT_DIR=$(readlink -f "$(dirname "$0")")
 WORKSPACE_ROOT="$SCRIPT_DIR/../"
 
+# Load env
+source "$WORKSPACE_ROOT/amd64.env"
+if [ "$(uname -m)" = "aarch64" ]; then
+    source "$WORKSPACE_ROOT/arm64.env"
+fi
+
 # https://github.com/docker/buildx/issues/484
 export BUILDKIT_STEP_LOG_MAX_SIZE=10000000
 
 docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/autoware-universe/docker-bake.hcl" \
     --set "*.context=$WORKSPACE_ROOT" \
     --set "*.ssh=default" \
+    --set "*.args.ROS_DISTRO=$rosdistro" \
+    --set "*.args.CUDA_IMAGE_TAG=$cuda_image_tag" \
+    --set "*.args.CUDNN_VERSION=$cudnn_version" \
+    --set "*.args.TENSORRT_VERSION=$tensorrt_version" \
     --set "devel.tags=ghcr.io/autowarefoundation/autoware-universe:latest" \
     --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt"

setup-dev-env.sh

@@ -70,6 +70,18 @@ elif [ "$option_yes" = "true" ]; then
     ansible_args+=("--extra-vars" "install_nvidia=y")
 fi
 
+# Load env
+source "$SCRIPT_DIR/amd64.env"
+if [ "$(uname -m)" = "aarch64" ]; then
+    source "$SCRIPT_DIR/arm64.env"
+fi
+
+# Add env args
+# shellcheck disable=SC2013
+for env_name in $(sed "s/=.*//" <amd64.env); do
+    ansible_args+=("--extra-vars" "${env_name}=${!env_name}")
+done
+
 # Install sudo
 if ! (command -v sudo >/dev/null 2>&1); then
     apt-get -y update