From c8be54d1c89714dd09b95f58430ed3f69f0f519b Mon Sep 17 00:00:00 2001 From: Kenji Miyake <31987104+kenji-miyake@users.noreply.github.com> Date: Mon, 25 Apr 2022 18:56:50 +0900 Subject: [PATCH] fix: clean up setup scripts to support ARM64 (#219) * fix: clean up setup scripts to support ARM64 Signed-off-by: Kenji Miyake * fix push condition Signed-off-by: Kenji Miyake * remove comment lines Signed-off-by: Kenji Miyake * rename env names Signed-off-by: Kenji Miyake * use context values Signed-off-by: Kenji Miyake * fix mistake Signed-off-by: Kenji Miyake * copy env files to Docker image Signed-off-by: Kenji Miyake * fix typo Signed-off-by: Kenji Miyake * fix bug Signed-off-by: Kenji Miyake * define Docker args after FROM Signed-off-by: Kenji Miyake --- .../actions/docker-build-and-push/action.yaml | 22 +++-------- .../docker-build-and-push-self-hosted.yaml | 14 ++++++- .github/workflows/docker-build-and-push.yaml | 14 ++++++- .shellcheckrc | 1 + amd64.env | 6 +++ ansible/playbooks/core.yaml | 9 +++-- ansible/playbooks/universe.yaml | 15 +++++--- ansible/roles/cuda/tasks/main.yaml | 9 ++++- .../roles/rmw_implementation/tasks/main.yaml | 8 ++-- arm64.env | 2 + docker/autoware-universe/Dockerfile | 37 ++++++++++--------- docker/build.sh | 10 +++++ setup-dev-env.sh | 12 ++++++ 13 files changed, 108 insertions(+), 51 deletions(-) create mode 100644 .shellcheckrc create mode 100644 amd64.env create mode 100644 arm64.env diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml index 62b628c6..cf8019a9 100644 --- a/.github/actions/docker-build-and-push/action.yaml +++ b/.github/actions/docker-build-and-push/action.yaml @@ -5,28 +5,16 @@ inputs: bake-target: description: "" required: true - platforms: + build-args: description: "" - required: true + required: false tag-suffix: description: "" - required: true + required: false runs: using: composite steps: - - name: Get branch name - id: get-branch-name - run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - shell: bash - - - name: Check branch of workflow_dispatch - if: ${{ github.event_name == 'workflow_dispatch' && steps.get-branch-name.outputs.branch != github.event.repository.default_branch }} - run: | - echo "workflow_dispatch is allowed only with the branch '${{ github.event.repository.default_branch }}', '${{ steps.get-branch-name.outputs.branch }}' is not allowed." - exit 1 - shell: bash - - name: Setup Docker Buildx uses: docker/setup-buildx-action@v1 @@ -104,9 +92,9 @@ runs: - name: Build and push uses: docker/bake-action@v1 with: - push: ${{ github.event_name != 'pull_request' }} + push: ${{ github.ref_name == github.event.repository.default_branch }} files: | docker/${{ inputs.bake-target }}/docker-bake.hcl bake.json set: | - *.platform=${{ inputs.platforms }} + ${{ inputs.build-args }} diff --git a/.github/workflows/docker-build-and-push-self-hosted.yaml b/.github/workflows/docker-build-and-push-self-hosted.yaml index be46b6ba..9dff6165 100644 --- a/.github/workflows/docker-build-and-push-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-self-hosted.yaml @@ -23,11 +23,23 @@ jobs: - name: Free disk space uses: ./.github/actions/free-disk-space + - name: Load env + run: | + cat amd64.env | sed "/^#/d" >> $GITHUB_ENV + if [ "$(uname -m)" = "aarch64" ]; then + cat arm64.env | sed "/^#/d" >> $GITHUB_ENV + fi + - name: Build 'autoware-universe' uses: ./.github/actions/docker-build-and-push with: bake-target: autoware-universe - platforms: linux/arm64 + build-args: | + *.platform=linux/arm64 + *.args.ROS_DISTRO=${{ env.rosdistro }} + *.args.CUDA_IMAGE_TAG=${{ env.cuda_image_tag }} + *.args.CUDNN_VERSION=${{ env.cudnn_version }} + *.args.TENSORRT_VERSION=${{ env.tensorrt_version }} tag-suffix: -arm64 - name: Show disk space diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml index 2616f634..a4012f39 100644 --- a/.github/workflows/docker-build-and-push.yaml +++ b/.github/workflows/docker-build-and-push.yaml @@ -18,11 +18,23 @@ jobs: - name: Free disk space uses: ./.github/actions/free-disk-space + - name: Load env + run: | + cat amd64.env | sed "/^#/d" >> $GITHUB_ENV + if [ "$(uname -m)" = "aarch64" ]; then + cat arm64.env | sed "/^#/d" >> $GITHUB_ENV + fi + - name: Build 'autoware-universe' uses: ./.github/actions/docker-build-and-push with: bake-target: autoware-universe - platforms: linux/amd64 + build-args: | + *.platform=linux/amd64 + *.args.ROS_DISTRO=${{ env.rosdistro }} + *.args.CUDA_IMAGE_TAG=${{ env.cuda_image_tag }} + *.args.CUDNN_VERSION=${{ env.cudnn_version }} + *.args.TENSORRT_VERSION=${{ env.tensorrt_version }} tag-suffix: -amd64 - name: Show disk space diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 00000000..8226afb6 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +external-sources=true diff --git a/amd64.env b/amd64.env new file mode 100644 index 00000000..204d6090 --- /dev/null +++ b/amd64.env @@ -0,0 +1,6 @@ +rosdistro=galactic +rmw_implementation=rmw_cyclonedds_cpp +cuda_version=11.4 +cuda_image_tag=11.4.1-devel-ubuntu20.04 +cudnn_version=8.2.4.15-1+cuda11.4 +tensorrt_version=8.2.4-1+cuda11.4 diff --git a/ansible/playbooks/core.yaml b/ansible/playbooks/core.yaml index a1e59646..cb0fcf73 100644 --- a/ansible/playbooks/core.yaml +++ b/ansible/playbooks/core.yaml @@ -1,8 +1,11 @@ - hosts: localhost connection: local - vars: - - rosdistro: galactic - - rmw_implementation: rmw_cyclonedds_cpp + pre_tasks: + - name: Print args + ansible.builtin.debug: + msg: + - rosdistro: "{{ rosdistro }}" + - rmw_implementation: "{{ rmw_implementation }}" roles: - role: autoware.dev_env.autoware_core - role: autoware.dev_env.docker diff --git a/ansible/playbooks/universe.yaml b/ansible/playbooks/universe.yaml index 15e8a0b3..f0dda075 100644 --- a/ansible/playbooks/universe.yaml +++ b/ansible/playbooks/universe.yaml @@ -1,11 +1,5 @@ - hosts: localhost connection: local - vars: - - rosdistro: galactic - - rmw_implementation: rmw_cyclonedds_cpp - - cuda_version: 11-4 - - cudnn_version: 8.2.4.15-1+cuda11.4 - - tensorrt_version: 8.2.4-1+cuda11.4 vars_prompt: - name: install_nvidia prompt: |- @@ -13,6 +7,15 @@ Did you confirm the licenses and are you sure to to install them? [y/N] private: false pre_tasks: + - name: Print args + ansible.builtin.debug: + msg: + - rosdistro: "{{ rosdistro }}" + - rmw_implementation: "{{ rmw_implementation }}" + - cuda_version: "{{ cuda_version }}" + - cudnn_version: "{{ cudnn_version }}" + - tensorrt_version: "{{ tensorrt_version }}" + - name: Show warnings that NVIDIA libraries will not be installed ansible.builtin.pause: seconds: 10 diff --git a/ansible/roles/cuda/tasks/main.yaml b/ansible/roles/cuda/tasks/main.yaml index fe7e0632..20ba8558 100644 --- a/ansible/roles/cuda/tasks/main.yaml +++ b/ansible/roles/cuda/tasks/main.yaml @@ -25,10 +25,15 @@ repo: deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/{{ cuda_architecture.stdout }}/ / state: present -- name: Install cuda-{{ cuda_version }} +- name: Get dash-case name of cuda_version + ansible.builtin.shell: bash -c 'sed -e "s/\./-/g" <<< $(echo {{ cuda_version }})' + register: dash_case_cuda_version + changed_when: false + +- name: Install cuda-{{ dash_case_cuda_version.stdout }} become: true ansible.builtin.apt: - name: cuda-{{ cuda_version }} + name: cuda-{{ dash_case_cuda_version.stdout }} update_cache: true - name: Add PATH to .bashrc diff --git a/ansible/roles/rmw_implementation/tasks/main.yaml b/ansible/roles/rmw_implementation/tasks/main.yaml index 49c3519d..1c5faea1 100644 --- a/ansible/roles/rmw_implementation/tasks/main.yaml +++ b/ansible/roles/rmw_implementation/tasks/main.yaml @@ -1,12 +1,12 @@ -- name: Get kebab-case name +- name: Get dash-case name of rmw_implementation ansible.builtin.shell: bash -c 'sed -e "s/_/-/g" <<< $(echo {{ rmw_implementation }})' - register: kebab_case_name + register: dash_case_rmw_implementation changed_when: false -- name: Install ros-{{ rosdistro }}-{{ kebab_case_name.stdout }} +- name: Install ros-{{ rosdistro }}-{{ dash_case_rmw_implementation.stdout }} become: true ansible.builtin.apt: - name: ros-{{ rosdistro }}-{{ kebab_case_name.stdout }} + name: ros-{{ rosdistro }}-{{ dash_case_rmw_implementation.stdout }} state: latest update_cache: true diff --git a/arm64.env b/arm64.env new file mode 100644 index 00000000..5094c097 --- /dev/null +++ b/arm64.env @@ -0,0 +1,2 @@ +# Override amd64's settings +cudnn_version=8.2.4.12-1+cuda11.4 diff --git a/docker/autoware-universe/Dockerfile b/docker/autoware-universe/Dockerfile index 5fbb0271..3f3b62cd 100644 --- a/docker/autoware-universe/Dockerfile +++ b/docker/autoware-universe/Dockerfile @@ -1,6 +1,11 @@ -FROM nvidia/cuda:11.4.1-devel-ubuntu20.04 as devel +ARG CUDA_IMAGE_TAG +FROM nvidia/cuda:$CUDA_IMAGE_TAG as devel SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ARG ROS_DISTRO +ARG CUDNN_VERSION +ARG TENSORRT_VERSION + ## Install apt packages # hadolint ignore=DL3008 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ @@ -10,7 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-ins && rm -rf /var/lib/apt/lists/* ## Copy files -COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml /autoware/ +COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ COPY ansible/ /autoware/ansible/ WORKDIR /autoware RUN ls /autoware @@ -25,30 +30,28 @@ RUN --mount=type=ssh \ && mkdir src \ && vcs import src < autoware.repos \ && rosdep update \ - && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro galactic \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* ## Install cuDNN and TensorRT -ARG CUDNN_VERSION=8.2.4.15-1+cuda11.4 -ARG TENSORRT_VERSION=8.2.4-1+cuda11.4 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ - libcudnn8=$CUDNN_VERSION \ - libcudnn8-dev=$CUDNN_VERSION \ - libnvinfer8=$TENSORRT_VERSION \ - libnvinfer-dev=$TENSORRT_VERSION \ - libnvinfer-plugin8=$TENSORRT_VERSION \ - libnvinfer-plugin-dev=$TENSORRT_VERSION \ - libnvonnxparsers8=$TENSORRT_VERSION \ - libnvonnxparsers-dev=$TENSORRT_VERSION \ - libnvparsers8=$TENSORRT_VERSION \ - libnvparsers-dev=$TENSORRT_VERSION \ + libcudnn8="$CUDNN_VERSION" \ + libcudnn8-dev="$CUDNN_VERSION" \ + libnvinfer8="$TENSORRT_VERSION" \ + libnvinfer-dev="$TENSORRT_VERSION" \ + libnvinfer-plugin8="$TENSORRT_VERSION" \ + libnvinfer-plugin-dev="$TENSORRT_VERSION" \ + libnvonnxparsers8="$TENSORRT_VERSION" \ + libnvonnxparsers-dev="$TENSORRT_VERSION" \ + libnvparsers8="$TENSORRT_VERSION" \ + libnvparsers-dev="$TENSORRT_VERSION" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* ## Create entrypoint # hadolint ignore=SC1091 -RUN echo "source /opt/ros/galactic/setup.bash" > /etc/bash.bashrc +RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc CMD ["/bin/bash"] FROM devel as prebuilt @@ -56,7 +59,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ## Build and change permission for runtime data conversion # hadolint ignore=SC1091 -RUN source /opt/ros/galactic/setup.bash \ +RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \ && find /autoware/install -type d -exec chmod 777 {} \; diff --git a/docker/build.sh b/docker/build.sh index 7149104c..bcfb42c3 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -5,11 +5,21 @@ set -e SCRIPT_DIR=$(readlink -f "$(dirname "$0")") WORKSPACE_ROOT="$SCRIPT_DIR/../" +# Load env +source "$WORKSPACE_ROOT/amd64.env" +if [ "$(uname -m)" = "aarch64" ]; then + source "$WORKSPACE_ROOT/arm64.env" +fi + # https://github.com/docker/buildx/issues/484 export BUILDKIT_STEP_LOG_MAX_SIZE=10000000 docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/autoware-universe/docker-bake.hcl" \ --set "*.context=$WORKSPACE_ROOT" \ --set "*.ssh=default" \ + --set "*.args.ROS_DISTRO=$rosdistro" \ + --set "*.args.CUDA_IMAGE_TAG=$cuda_image_tag" \ + --set "*.args.CUDNN_VERSION=$cudnn_version" \ + --set "*.args.TENSORRT_VERSION=$tensorrt_version" \ --set "devel.tags=ghcr.io/autowarefoundation/autoware-universe:latest" \ --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt" diff --git a/setup-dev-env.sh b/setup-dev-env.sh index fa18b82e..3cadaef8 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -70,6 +70,18 @@ elif [ "$option_yes" = "true" ]; then ansible_args+=("--extra-vars" "install_nvidia=y") fi +# Load env +source "$SCRIPT_DIR/amd64.env" +if [ "$(uname -m)" = "aarch64" ]; then + source "$SCRIPT_DIR/arm64.env" +fi + +# Add env args +# shellcheck disable=SC2013 +for env_name in $(sed "s/=.*//" /dev/null 2>&1); then apt-get -y update