Improve RHEL container files

- Update to Python 3.10.14, which is currently the latest security release of Python 3.10 series. - Use crypto policy `DEFAULT:SHA1` instead of `LEGACY` to work around Habana GPG key with SHA-1 signature. It's less insecure than `LEGACY`. - Enable GPG checks for CentOS repositories. - Only install 64bit Intel oneMKL packages. The 32bit packages are not needed. - Use `$(DOCKER)` variable in Makefile for PyTorch container, so `make DOCKER=podman` works correctly. - Explicitly set `PYTHON_VERSION`, so the package installer uses a fully qualified Python binary (e.g. `python3.10`) for installation. - Use a `.pth` file instead of `PYTHONPATH` env var Signed-off-by: Christian Heimes <[email protected]>
HabanaAI · Jun 24, 2024 · a16411c · a16411c
1 parent fe01c25
commit a16411c
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 17 deletions.
diff --git a/dockerfiles/base/Dockerfile.rhel8.6 b/dockerfiles/base/Dockerfile.rhel8.6
@@ -19,13 +19,15 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n
 RUN echo "[appstream]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
     echo "name=CentOS Linux 8 - AppStream" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
     echo "baseurl=https://vault.centos.org/8-stream/AppStream/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
 
 
 RUN echo "[BaseOS]" > /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
     echo "name=CentOS Linux 8 - BaseOS" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
     echo "baseurl=https://vault.centos.org/8-stream/BaseOS/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
 
 RUN dnf install -y \
     clang \
@@ -78,7 +80,8 @@ RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
 RUN echo "[powertools]" > /etc/yum.repos.d/powertools.repo && \
     echo "name=powertools" >> /etc/yum.repos.d/powertools.repo && \
     echo "baseurl=https://vault.centos.org/8-stream/PowerTools/x86_64/os/"  >> /etc/yum.repos.d/powertools.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/powertools.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/powertools.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/powertools.repo
 
 RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el8 \
         habanalabs-thunk-"$VERSION"-"$REVISION".el8 \
@@ -108,6 +111,7 @@ RUN wget -nv -O /tmp/main.zip https://github.com/HabanaAI/hccl_ofi_wrapper/archi
     cd / && \
     rm -rf /tmp/main.zip /tmp/hccl_ofi_wrapper-main
 
+ENV PYTHON_VERSION=3.8
 RUN python3.8 -m pip install pip==23.3.1 setuptools==67.3.3 wheel==0.38.4
 
 RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 && \

diff --git a/dockerfiles/base/Dockerfile.rhel9.2 b/dockerfiles/base/Dockerfile.rhel9.2
@@ -19,12 +19,14 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.n
 RUN echo "[BaseOS]" > /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
     echo "name=CentOS Linux 9 - BaseOS" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
     echo "baseurl=https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
 
 RUN echo "[centos9]" > /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
     echo "name=CentOS Linux 9 - AppStream" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
     echo "baseurl=https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-AppStream.repo
 
 RUN dnf install -y \
     clang \
@@ -61,6 +63,7 @@ RUN dnf install -y \
     dnf update -y && \
     dnf clean all && rm -rf /var/cache/yum
 
+ENV PYTHON_VERSION=3.10
 COPY install-python310.sh .
 RUN ./install-python310.sh rhel9.2 && rm install-python310.sh
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
@@ -84,7 +87,8 @@ RUN echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \
     echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.2/repodata/repomd.xml.key" >> /etc/yum.repos.d/habanalabs.repo && \
     echo 'gpgcheck=1' >> /etc/yum.repos.d/habanalabs.repo
 
-RUN update-crypto-policies --set LEGACY
+# for Habana GPG key with SHA-1 signature
+RUN update-crypto-policies --set DEFAULT:SHA1
 
 RUN dnf install -y habanalabs-rdma-core-"$VERSION"-"$REVISION".el9 \
     habanalabs-thunk-"$VERSION"-"$REVISION".el9 \

diff --git a/dockerfiles/base/install-python310.sh b/dockerfiles/base/install-python310.sh
@@ -35,11 +35,11 @@ case "${_BASE_NAME}" in
 esac
 
 # install Python
-wget -nv -O /opt/Python-3.10.9.tgz https://www.python.org/ftp/python/3.10.9/Python-3.10.9.tgz
+wget -nv -O /opt/Python-3.10.14.tgz https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz
 cd /opt/
-tar xzf Python-3.10.9.tgz
-rm -f Python-3.10.9.tgz
-cd Python-3.10.9
+tar xzf Python-3.10.14.tgz
+rm -f Python-3.10.14.tgz
+cd Python-3.10.14
 ./configure --enable-optimizations --enable-loadable-sqlite-extensions --enable-shared $_SSL_LIB
 make -j && make altinstall
 

diff --git a/dockerfiles/pytorch/Dockerfile.rhel8.6 b/dockerfiles/pytorch/Dockerfile.rhel8.6
@@ -17,7 +17,7 @@ LABEL name="PyTorch Installer"
 LABEL summary="Habanalabs PyTorch installer layer for RHEL8.6"
 LABEL description="Image with pre installed Habanalabs packages for PyTorch"
 
-ENV PYTHONPATH=/root:/usr/lib/habanalabs/
+RUN echo "/usr/lib/habanalabs" > $(python3.8 -c "import sysconfig; print(sysconfig.get_path('platlib'))")/habanalabs-graph.pth
 
 RUN dnf install -y \
     curl \
@@ -34,7 +34,7 @@ RUN dnf install -y \
     dnf clean all && rm -rf /var/cache/yum
 
 RUN dnf config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -y && \
-    dnf install --allowerasing -y intel-mkl-2020.4-912 && \
+    dnf install --allowerasing -y intel-mkl-64bit-2020.4-912 && \
     dnf clean all && rm -rf /var/cache/yum
 
 COPY install_packages.sh .

diff --git a/dockerfiles/pytorch/Dockerfile.rhel9.2 b/dockerfiles/pytorch/Dockerfile.rhel9.2
@@ -17,12 +17,13 @@ LABEL name="PyTorch Installer"
 LABEL summary="Habanalabs PyTorch installer layer for RHEL9.2"
 LABEL description="Image with pre installed Habanalabs packages for PyTorch"
 
-ENV PYTHONPATH=/root:/usr/lib/habanalabs/
+RUN echo "/usr/lib/habanalabs" > $(python3.10 -c "import sysconfig; print(sysconfig.get_path('platlib'))")/habanalabs-graph.pth
 
 RUN echo "[CRB]" > /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
     echo "name=CentOS Linux 9 - CRB" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
     echo "baseurl=https://mirror.stream.centos.org/9-stream/CRB/x86_64/os" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
-    echo "gpgcheck=0" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo
+    echo "gpgkey=https://www.centos.org/keys/RPM-GPG-KEY-CentOS-Official-SHA256" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo && \
+    echo "gpgcheck=1" >> /etc/yum.repos.d/CentOS-Linux-CRB.repo
 
 RUN dnf install --allowerasing -y \
     curl \
@@ -38,7 +39,7 @@ RUN dnf install --allowerasing -y \
     dnf clean all && rm -rf /var/cache/yum
 
 RUN dnf config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -y && \
-    dnf install --allowerasing -y intel-mkl-2020.4-912 && \
+    dnf install --allowerasing -y intel-mkl-64bit-2020.4-912 && \
     dnf clean all && rm -rf /var/cache/yum
 
 COPY install_packages.sh .

diff --git a/dockerfiles/pytorch/Dockerfile.tencentos3.1 b/dockerfiles/pytorch/Dockerfile.tencentos3.1
@@ -34,7 +34,7 @@ RUN dnf install -y \
     dnf clean all && rm -rf /var/cache/yum
 
 RUN dnf config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -y && \
-    dnf install --allowerasing -y intel-mkl-2020.4-912 && \
+    dnf install --allowerasing -y intel-mkl-64bit-2020.4-912 && \
     dnf clean all && rm -rf /var/cache/yum
 
 COPY install_packages.sh .

diff --git a/dockerfiles/pytorch/Makefile b/dockerfiles/pytorch/Makefile
@@ -5,7 +5,7 @@ IMAGE_NAME = pytorch-installer-${BUILD_OS}-$(PT_VERSION)
 DOCKER_BUILD_ARGS := $(DOCKER_BUILD_ARGS) --build-arg PT_VERSION=$(PT_VERSION)
 
 base:
-ifneq ($(shell docker image inspect $(BASE_IMAGE_URL):$(RELEASE_VERSION)-$(RELEASE_BUILD_ID) --format="image_exists" 2>/dev/null), image_exists)
+ifneq ($(shell $(DOCKER) image inspect $(BASE_IMAGE_URL):$(RELEASE_VERSION)-$(RELEASE_BUILD_ID) --format="image_exists" 2>/dev/null), image_exists)
 	cd ../base; \
 	make build; \
 	cd ../pytorch