Merge pull request openucx#8894 from Al3xR01/topic/arm_release-part1

AZP/RELEASE: Add ARM release - Part #1
changchengx · Apr 4, 2023 · 96d0c17 · 96d0c17
2 parents b67d582 + 96eb517
commit 96d0c17
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 17 deletions.
diff --git a/buildlib/dockers/centos-release.Dockerfile b/buildlib/dockers/centos-release.Dockerfile
@@ -31,7 +31,8 @@ RUN yum install -y \
 # MOFED
 ARG MOFED_VERSION
 ARG MOFED_OS
-ENV MOFED_DIR MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64
+ARG ARCH
+ENV MOFED_DIR MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-${ARCH}
 ENV MOFED_SITE_PLACE MLNX_OFED-${MOFED_VERSION}
 ENV MOFED_IMAGE ${MOFED_DIR}.tgz
 RUN wget --no-verbose http://content.mellanox.com/ofed/${MOFED_SITE_PLACE}/${MOFED_IMAGE} && \

diff --git a/buildlib/dockers/docker-compose-aarch64.yml b/buildlib/dockers/docker-compose-aarch64.yml
@@ -0,0 +1,59 @@
+version: "3.4"
+# To build images, run:
+# docker compose -f docker-compose-aarch64.yml build
+
+# Find MOFED_VERSION per OS and arch:
+# https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/
+
+# Find NV_DRIVER_VERSION per CUDA version, OS and arch:
+# https://developer.nvidia.com/cuda-11-7-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=runfile_local
+
+services:
+  centos8-mofed5-cuda11:
+    image: centos8-mofed5-cuda11:5
+    build:
+      context: .
+      network: host
+      dockerfile: centos-release.Dockerfile
+      args:
+        MOFED_VERSION: 5.4-3.6.8.1
+        MOFED_OS: rhel8.2
+        CUDA_VERSION: 11.4.0
+        OS_VERSION: 8
+        ARCH: aarch64
+  ubuntu18.04-mofed5-cuda11:
+    image: ubuntu18.04-mofed5-cuda11:5
+    build:
+      context: .
+      network: host
+      dockerfile: ubuntu-release.Dockerfile
+      args:
+        MOFED_VERSION: 5.4-3.6.8.1
+        UBUNTU_VERSION: 18.04
+        CUDA_VERSION: 11.4.0
+        CUDA_COMPAT: 11-4
+        ARCH: aarch64
+  ubuntu20.04-mofed5-cuda11:
+    image: ubuntu20.04-mofed5-cuda11:5
+    build:
+      context: .
+      network: host
+      dockerfile: ubuntu-release.Dockerfile
+      args:
+        MOFED_VERSION: 5.4-3.6.8.1
+        UBUNTU_VERSION: 20.04
+        CUDA_VERSION: 11.4.0
+        CUDA_COMPAT: 11-4
+        ARCH: aarch64
+  ubuntu22.04-mofed5-cuda11:
+    image: ubuntu22.04-mofed5-cuda11:5
+    build:
+      context: .
+      network: host
+      dockerfile: ubuntu-release.Dockerfile
+      args:
+        MOFED_VERSION: 5.4-3.6.8.1
+        UBUNTU_VERSION: 22.04
+        CUDA_VERSION: 11.7.0
+        CUDA_COMPAT: 11-7
+        ARCH: aarch64
diff --git a/buildlib/dockers/docker-compose.yml → buildlib/dockers/docker-compose-x86_64.yml b/buildlib/dockers/docker-compose.yml → buildlib/dockers/docker-compose-x86_64.yml
@@ -1,11 +1,16 @@
 version: "3.4"
+# To build images, run:
+# docker compose -f docker-compose-x86_64.yml build
 
-# Find driver version based on CUDA version, OS and CPU arch (515 in this case):
+# Find MOFED_VERSION per OS and arch:
+# https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/
+
+# Find NV_DRIVER_VERSION per CUDA version, OS and arch:
 # https://developer.nvidia.com/cuda-11-7-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=runfile_local
 
 services:
   centos7-mofed5-cuda11:
-    image: centos7-mofed5-cuda11:2
+    image: centos7-mofed5-cuda11:5
     build:
       context: .
       network: host
@@ -15,8 +20,9 @@ services:
         MOFED_OS: rhel7.6
         CUDA_VERSION: 11.4.0
         OS_VERSION: 7
+        ARCH: x86_64
   centos7-mofed5.4-cuda11:
-    image: centos7-mofed5.4-cuda11
+    image: centos7-mofed5.4-cuda11:5
     build:
       context: .
       network: host
@@ -26,8 +32,9 @@ services:
         MOFED_OS: rhel7.6
         CUDA_VERSION: 11.2.0
         OS_VERSION: 7
+        ARCH: x86_64
   centos8-mofed5-cuda11:
-    image: centos8-mofed5-cuda11:2
+    image: centos8-mofed5-cuda11:5
     build:
       context: .
       network: host
@@ -37,8 +44,20 @@ services:
         MOFED_OS: rhel8.2
         CUDA_VERSION: 11.4.0
         OS_VERSION: 8
+        ARCH: x86_64
+  ubuntu16.04-mofed5-cuda11:
+    image: ubuntu16.04-mofed5-cuda11:5
+    build:
+      context: .
+      network: host
+      dockerfile: ubuntu-release.Dockerfile
+      args:
+        MOFED_VERSION: 5.0-1.0.0.0
+        UBUNTU_VERSION: 16.04
+        CUDA_VERSION: 11.2.0
+        ARCH: x86_64
   ubuntu18.04-mofed5-cuda11:
-    image: ubuntu18.04-mofed5-cuda11:3
+    image: ubuntu18.04-mofed5-cuda11:5
     build:
       context: .
       network: host
@@ -47,9 +66,9 @@ services:
         MOFED_VERSION: 5.0-1.0.0.0
         UBUNTU_VERSION: 18.04
         CUDA_VERSION: 11.4.0
-        NV_DRIVER_VERSION: 470
+        ARCH: x86_64
   ubuntu20.04-mofed5-cuda11:
-    image: ubuntu20.04-mofed5-cuda11:3
+    image: ubuntu20.04-mofed5-cuda11:5
     build:
       context: .
       network: host
@@ -58,9 +77,9 @@ services:
         MOFED_VERSION: 5.0-1.0.0.0
         UBUNTU_VERSION: 20.04
         CUDA_VERSION: 11.4.0
-        NV_DRIVER_VERSION: 470
+        ARCH: x86_64
   ubuntu22.04-mofed5-cuda11:
-    image: ubuntu22.04-mofed5-cuda11:3
+    image: ubuntu22.04-mofed5-cuda11:5
     build:
       context: .
       network: host
@@ -69,4 +88,4 @@ services:
         MOFED_VERSION: 5.4-3.6.8.1
         UBUNTU_VERSION: 22.04
         CUDA_VERSION: 11.7.0
-        NV_DRIVER_VERSION: 515
+        ARCH: x86_64
diff --git a/buildlib/dockers/push-release-images.sh b/buildlib/dockers/push-release-images.sh
@@ -2,8 +2,8 @@
 
 # shellcheck disable=SC2086
 basedir=$(cd "$(dirname $0)" && pwd)
-
-registry=harbor.mellanox.com/ucx
+ARCH=$(uname -m)
+registry=harbor.mellanox.com/ucx/${ARCH}
 
 images=$(awk '!/#/ && /image:/ {print $2}' "${basedir}/docker-compose.yml")
 for img in $images; do

diff --git a/buildlib/dockers/ubuntu-release.Dockerfile b/buildlib/dockers/ubuntu-release.Dockerfile
@@ -16,23 +16,23 @@ RUN apt-get update && \
         libcap2 \
         libnuma-dev \
         libtool \
-        # Provide the dependencies required by libnvidia-compute* instead the cuda-compat*
+        # Provide CUDA dependencies by libnvidia-compute*
         libnvidia-compute-${NV_DRIVER_VERSION} \
         make \
         maven \
         udev \
         wget \
         environment-modules \
-        pkg-config \
-        sudo \
+    # Remove cuda-compat* from nvidia/cuda:x86_64 images, provide CUDA dependencies by libnvidia-compute* instead
     && apt-get remove -y openjdk-11-* cuda-compat* || apt-get autoremove -y \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # MOFED
 ARG MOFED_VERSION=5.0-1.0.0.0
 ARG UBUNTU_VERSION
 ARG MOFED_OS=ubuntu${UBUNTU_VERSION}
-ENV MOFED_DIR MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64
+ARG ARCH
+ENV MOFED_DIR MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-${ARCH}
 ENV MOFED_SITE_PLACE MLNX_OFED-${MOFED_VERSION}
 ENV MOFED_IMAGE ${MOFED_DIR}.tgz
 RUN wget --no-verbose http://content.mellanox.com/ofed/${MOFED_SITE_PLACE}/${MOFED_IMAGE} && \