fix CI failure when building manylinux and dev images

alibaba · Aug 12, 2024 · 87f8fcb · 87f8fcb
1 parent 6e15ef7
commit 87f8fcb
Show file tree

Hide file tree

Showing 22 changed files with 393 additions and 506 deletions.
diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml
diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml
@@ -0,0 +1,56 @@
+name: Build GraphScope Wheel Images
+
+# build `graphscope/graphscope-dev:wheel-{v6d_version}` image based on manylinux,
+# including all dependencies for building graphscope wheel package.
+on:
+  workflow_dispatch:
+    inputs:
+      v6d_version:
+        description: 'Version for Vineyard (v6d)'
+        required: true
+        default: 'main'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'python/graphscope/gsctl/scripts/**'
+      - '.github/workflows/build-graphscope-dev-wheel-images.yml'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: registry.cn-hongkong.aliyuncs.com
+
+jobs:
+  build-wheel-image-amd64:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Build Image
+      run: |
+        # build wheel image with specified v6d's version
+        cd ${GITHUB_WORKSPACE}/k8s
+        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
+        if [[ -n ${VINEYARD_VERSION} ]]; then
+          # graphscope/graphscope-dev:wheel-<v6d_version>-amd64
+          make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
+        else
+          # pull_request: use default vineyard_version
+          make dev-wheel
+        fi
+
+    - name: Release Image
+      if: ${{ github.event_name == 'workflow_dispatch' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
+        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
+        sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml
diff --git a/.github/workflows/build-graphscope-manylinux-images.yml b/.github/workflows/build-graphscope-manylinux-images.yml
@@ -0,0 +1,45 @@
+name: Build GraphScope Manylinux Images
+
+# build `graphscope/manylinux2014` image based on centos7, including all necessary
+# dependencies except vineyard
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'python/graphscope/gsctl/scripts/**'
+      - '.github/workflows/build-graphscope-manylinux-images.yml'
+
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: registry.cn-hongkong.aliyuncs.com
+
+jobs:
+  build-manylinux-image-amd64:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Build Image
+      run: |
+        cd ${GITHUB_WORKSPACE}/k8s
+        # output: graphscope/manylinux:amd64
+        make manylinux2014
+
+    - name: Release Image
+      if: ${{ github.event_name == 'workflow_dispatch' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
+        # manylinux2014 image
+        sudo docker tag graphscope/manylinux2014:amd64 ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64
diff --git a/k8s/Makefile b/k8s/Makefile
@@ -6,8 +6,10 @@ ifeq ($(REGISTRY),)
     REGISTRY := registry.cn-hongkong.aliyuncs.com
 endif
 
-# x86_64 or aarch64
-ARCH := $(shell uname -m)
+PLATFORM          := $(shell uname -m)
+# can be: x86_64, arm64
+ARCH              := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM)))
+
 
 VERSION ?= latest
 VINEYARD_VERSION ?= v0.23.0
@@ -41,36 +43,34 @@ interactive: interactive-frontend interactive-executor
 all: coordinator analytical interactive learning
 graphscope: all
 
+manylinux2014:
+	cd $(WORKING_DIR)/../ && \
+	docker build \
+		-t graphscope/manylinux2014:${ARCH} \
+		-f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile .
+
 dev-wheel:
-	cd $(WORKING_DIR) && \
-	cd ../python/graphscope && \
+	cd $(WORKING_DIR)/../ && \
 	docker build \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \
 		-t graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \
-		-f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile .
-
-
-manylinux2014-ext:
-	cd $(WORKING_DIR) && \
-	cd ../python/graphscope  && \
-	docker build \
-		-t graphscope/manylinux2014:ext-${ARCH} \
-		-f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile .
+		-f $(DOCKERFILES_DIR)/graphscope-dev-wheel.Dockerfile .
 
 graphscope-dev:
-	cd $(WORKING_DIR) && \
-	cd ../python/graphscope  && \
+	cd $(WORKING_DIR)/../ && \
 	docker build \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \
 		-t graphscope/graphscope-dev:${VINEYARD_VERSION}-${ARCH} \
 		-f $(DOCKERFILES_DIR)/graphscope-dev.Dockerfile .
 
 vineyard-dev:
-	cd $(WORKING_DIR) && \
-	cd ../python/graphscope  && \
+	cd $(WORKING_DIR)/../ && \
 	docker build \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \
 		-t graphscope/vineyard-dev:${VINEYARD_VERSION}-${ARCH} \
@@ -79,6 +79,7 @@ vineyard-dev:
 vineyard-runtime:
 	cd $(WORKING_DIR) && \
 	docker build \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		-t graphscope/vineyard-runtime:${VINEYARD_VERSION}-${ARCH} \
@@ -88,6 +89,7 @@ coordinator:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target coordinator \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg CI=${CI} \
@@ -98,6 +100,7 @@ analytical:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target analytical \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -109,6 +112,7 @@ analytical-java:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target analytical-java \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -120,6 +124,7 @@ interactive-frontend:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target frontend \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -132,6 +137,7 @@ interactive-executor:
 	cd $(WORKING_DIR)/.. \
 	&& docker build \
 	    --target executor \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -145,6 +151,7 @@ interactive-experimental:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target experimental \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		-t graphscope/interactive-experimental:${VERSION} \
@@ -155,6 +162,7 @@ flex-interactive:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 		--target runtime \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
@@ -166,6 +174,7 @@ learning:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target learning \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -177,6 +186,7 @@ graphlearn-torch:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
 	    --target graphlearn-torch \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \
@@ -187,6 +197,7 @@ graphlearn-torch:
 graphscope-store:
 	cd $(WORKING_DIR)/.. && \
 	docker build \
+		--build-arg ARCH=$(ARCH) \
 		--build-arg REGISTRY=$(REGISTRY) \
 		--build-arg BUILDER_VERSION=$(BUILDER_VERSION) \
 		--build-arg profile=${PROFILE} \