From 516c542fb56c6966726d6308508680f0532b1e7c Mon Sep 17 00:00:00 2001 From: yecol Date: Wed, 28 Feb 2024 14:08:35 +0800 Subject: [PATCH] testing spellcheck. --- .../workflows/build-graphscope-dev-images.yml | 472 ----------- .../build-graphscope-images-linux.yml | 458 ----------- .../build-graphscope-manylinux-ext-images.yml | 106 --- .../build-graphscope-wheels-linux.yml | 359 --------- .../build-graphscope-wheels-macos.yml | 327 -------- .github/workflows/docs.yml | 266 ------- .github/workflows/flex-dummy.yml | 21 - .github/workflows/flex.yml | 204 ----- .github/workflows/gae-dummy.yml | 23 - .github/workflows/gae.yml | 105 --- .github/workflows/gaia.yml | 113 --- .github/workflows/gss-dummy.yml | 39 - .github/workflows/gss.yml | 239 ------ .github/workflows/hqps-db-ci.yml | 223 ------ .github/workflows/k8s-ci-dummy.yml | 115 --- .github/workflows/k8s-ci.yml | 743 ------------------ .github/workflows/local-ci-dummy.yml | 106 --- .github/workflows/local-ci.yml | 577 -------------- .../networkx-forward-algo-nightly.yml | 53 -- .github/workflows/nightly.yml | 78 -- .github/workflows/pegasus.yml | 84 -- .github/workflows/release.yml | 242 ------ .github/workflows/spell-check.yml | 11 + .github/workflows/stale.yaml | 89 --- learning_engine/graph-learn | 2 +- 25 files changed, 12 insertions(+), 5043 deletions(-) delete mode 100644 .github/workflows/build-graphscope-dev-images.yml delete mode 100644 .github/workflows/build-graphscope-images-linux.yml delete mode 100644 .github/workflows/build-graphscope-manylinux-ext-images.yml delete mode 100644 .github/workflows/build-graphscope-wheels-linux.yml delete mode 100644 .github/workflows/build-graphscope-wheels-macos.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/flex-dummy.yml delete mode 100644 .github/workflows/flex.yml delete mode 100644 .github/workflows/gae-dummy.yml delete mode 100644 .github/workflows/gae.yml delete mode 100644 .github/workflows/gaia.yml delete mode 100644 .github/workflows/gss-dummy.yml delete mode 100644 .github/workflows/gss.yml delete mode 100644 .github/workflows/hqps-db-ci.yml delete mode 100644 .github/workflows/k8s-ci-dummy.yml delete mode 100644 .github/workflows/k8s-ci.yml delete mode 100644 .github/workflows/local-ci-dummy.yml delete mode 100644 .github/workflows/local-ci.yml delete mode 100644 .github/workflows/networkx-forward-algo-nightly.yml delete mode 100644 .github/workflows/nightly.yml delete mode 100644 .github/workflows/pegasus.yml delete mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/spell-check.yml delete mode 100644 .github/workflows/stale.yaml diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml deleted file mode 100644 index 944ce796ac00..000000000000 --- a/.github/workflows/build-graphscope-dev-images.yml +++ /dev/null @@ -1,472 +0,0 @@ -name: Build GraphScope Development and Wheel Images - -# build images for: -# 1) wheel: including all dependencies for graphscope's wheel package. -# 2) graphscope-dev: including all dependencies for graphscope's development env. -# 3) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. -# 4) vineyard-runtime: including all vineyard-related running dependencies. -# Note that: -# Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github. -on: - workflow_dispatch: - inputs: - v6d_version: - description: 'Version for Vineyard (v6d)' - required: true - default: 'main' - build_wheel: - description: 'Whether to build dev-wheel image' - required: true - default: true - type: boolean - build_graphscope_dev: - description: 'Whether to build graphscope-dev image' - required: true - default: true - type: boolean - build_vineyard_dev: - description: 'Whether to build vineyard-dev(runtime) image' - required: true - default: true - type: boolean - pull_request: - branches: - - main - paths: - - 'python/graphscope/gsctl/**' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-wheel-image-x86-64: - runs-on: ubuntu-20.04 - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.pull_request == 'pull_request') - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--x86_64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - build-wheel-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.pull_request == 'pull_request') - if: false - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--aarch64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:wheel-${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} || true - - manifest-push-wheel-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_wheel == 'true' }} - if: false - - needs: [build-wheel-image-x86-64, build-wheel-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} - - build-graphscope-dev-image-x86-64: - runs-on: ubuntu-20.04 - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.pull_request == 'pull_request') - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build graphscope dev image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-x86_64 - make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make graphscope-dev - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - - build-graphscope-dev-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.pull_request == 'pull_request') - if: false - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build graphscope dev image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-aarch64 - make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make graphscope-dev - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} || true - - manifest-push-graphscope-dev-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_graphscope_dev == 'true' }} - if: false - - needs: [build-graphscope-dev-image-x86-64, build-graphscope-dev-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} - - build-vineyard-dev-image-x86-64: - runs-on: ubuntu-20.04 - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.pull_request == 'pull_request') - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard dev image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-dev:-x86_64 - make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-dev - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - - build-vineyard-dev-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.pull_request == 'pull_request') - if: false - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard dev image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-dev:-aarch64 - make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-dev - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/vineyard-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} || true - - manifest-push-vineyard-dev-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} - if: false - - needs: [build-vineyard-dev-image-x86-64, build-vineyard-dev-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} - - build-vineyard-runtime-image-x86-64: - runs-on: ubuntu-20.04 - # only trigger this step in 'workflow_dispatch' event, - # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' - if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} - - needs: [manifest-push-vineyard-dev-image] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard runtime image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-runtime:-x86_64 - make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-runtime - fi - - - name: Release Image - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-runtime image - sudo docker tag graphscope/vineyard-runtime:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - - build-vineyard-runtime-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # only trigger this step in 'workflow_dispatch' event, - # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} - if: false - - needs: [manifest-push-vineyard-dev-image] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard dev image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-runtime:-aarch64 - make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-runtime - fi - - - name: Release Image - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # vineyard-runtime image - sudo docker tag graphscope/vineyard-runtime:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/vineyard-runtime:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} || true - - manifest-push-vineyard-runtime-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} - if: false - - needs: [build-vineyard-runtime-image-x86-64, build-vineyard-runtime-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} diff --git a/.github/workflows/build-graphscope-images-linux.yml b/.github/workflows/build-graphscope-images-linux.yml deleted file mode 100644 index f9c8ad7eabaf..000000000000 --- a/.github/workflows/build-graphscope-images-linux.yml +++ /dev/null @@ -1,458 +0,0 @@ -name: Build GraphScope Images on Linux - -# on: [push, pull_request] -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - push: - tags: - - "v*" - pull_request: - branches: - - main - paths: - - 'k8s/dockerfiles/analytical.Dockerfile' - - 'k8s/dockerfiles/coordinator.Dockerfile' - - 'k8s/dockerfiles/interactive.Dockerfile' - - 'k8s/dockerfiles/learning.Dockerfile' - - '.github/workflows/build-graphscope-images-linux.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-image-x86-64: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.pull_request == 'pull_request') || (github.event_name == 'workflow_dispatch') - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build GraphScope Image - run: | - bash ${GITHUB_WORKSPACE}/.github/free-disk-space.sh || true - - cd ${GITHUB_WORKSPACE}/k8s - df -h - make coordinator CI=false VERSION=${SHORT_SHA} - df -h - make analytical CI=false VERSION=${SHORT_SHA} - df -h - # skip build analytical-java - # make analytical-java CI=false VERSION=${SHORT_SHA} - make interactive-frontend CI=false VERSION=${SHORT_SHA} - df -h - make interactive-executor CI=false VERSION=${SHORT_SHA} - df -h - make learning CI=false VERSION=${SHORT_SHA} - df -h - - - name: Release Nightly Image - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - # arch=$(uname -m) - # docker tag: 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}" - - # graphscope image - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # Don't push analytical-java, since it's relatively large and not used at this moment. - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - # dataset image - # Note! dataset image are built manually just use the latest one. - sudo docker pull ${{ env.REGISTRY }}/graphscope/dataset:latest - sudo docker tag ${{ env.REGISTRY }}/graphscope/dataset:latest ${{ env.REGISTRY }}/graphscope/dataset:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/dataset:${tag} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - # arch=$(uname -m) - # Release version tag - tag=${{ steps.tag.outputs.TAG }} - # graphscope image - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - # dataset image - # Note! dataset image are built manually just use the latest one. - sudo docker pull ${{ env.REGISTRY }}/graphscope/dataset:latest - sudo docker tag ${{ env.REGISTRY }}/graphscope/dataset:latest ${{ env.REGISTRY }}/graphscope/dataset:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/dataset:${tag} - - # Release the latest tag - tag=latest - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - - build-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.pull_request == 'pull_request') || (github.event_name == 'workflow_dispatch') - if: false - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build GraphScope Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - make coordinator CI=false VERSION=${SHORT_SHA} - make analytical CI=false VERSION=${SHORT_SHA} - # skip build analytical-java - # make analytical-java CI=false VERSION=${SHORT_SHA} - make interactive-frontend CI=false VERSION=${SHORT_SHA} - make interactive-executor CI=false VERSION=${SHORT_SHA} - make learning CI=false VERSION=${SHORT_SHA} - - - name: Release Nightly Image - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # docker tag: 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - - # graphscope image - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # Don't push analytical-java, since it's relatively large and not used at this moment. - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # Release version tag - tag=${{ steps.tag.outputs.TAG }}-${arch} - # graphscope image - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - # Release the latest tag - tag=latest-${arch} - sudo docker tag graphscope/coordinator:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker tag graphscope/analytical:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker tag graphscope/analytical-java:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker tag graphscope/interactive-frontend:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker tag graphscope/interactive-executor:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker tag graphscope/learning:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/learning:${tag} - - sudo docker push ${{ env.REGISTRY }}/graphscope/coordinator:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/analytical:${tag} - # skip build analytical-java - # sudo docker push ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/learning:${tag} - - - name: Clean Nightly Image - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - run: | - # aarch64 - arch=$(uname -m) - # docker tag: 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - # clean - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/coordinator:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true - tag=${SHORT_SHA} - sudo docker rmi -f graphscope/coordinator:${tag} || true - sudo docker rmi -f graphscope/analytical:${tag} || true - sudo docker rmi -f graphscope/analytical-java:${tag} || true - sudo docker rmi -f graphscope/interactive-frontend:${tag} || true - sudo docker rmi -f graphscope/interactive-executor:${tag} || true - sudo docker rmi -f graphscope/learning:${tag} || true - - - name: Clean Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - run: | - # aarch64 - arch=$(uname -m) - # Release version tag - tag=${{ steps.tag.outputs.TAG }}-${arch} - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/coordinator:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true - tag=latest-${arch} - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/coordinator:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/analytical-java:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-frontend:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/interactive-executor:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/learning:${tag} || true - tag=${SHORT_SHA} - sudo docker rmi -f graphscope/coordinator:${tag} || true - sudo docker rmi -f graphscope/analytical:${tag} || true - sudo docker rmi -f graphscope/analytical-java:${tag} || true - sudo docker rmi -f graphscope/interactive-frontend:${tag} || true - sudo docker rmi -f graphscope/interactive-executor:${tag} || true - sudo docker rmi -f graphscope/learning:${tag} || true - - - manifest-push-image: - runs-on: ubuntu-20.04 - # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - if: false - needs: [build-image-x86-64, build-image-aarch64] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Create and Push Docker Nightly Manifest - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - # create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/coordinator:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/coordinator:${version}a${time}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/coordinator:${version}a${time}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/analytical:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/analytical:${version}a${time}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/analytical:${version}a${time}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${version}a${time}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${version}a${time}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${version}a${time}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${version}a${time}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/learning:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/learning:${version}a${time}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/learning:${version}a${time}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/dataset:${version}a${time} \ - ${{ env.REGISTRY }}/graphscope/dataset:${version}a${time}-x86_64 - - # push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/coordinator:${version}a${time} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/analytical:${version}a${time} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${version}a${time} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:${version}a${time} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:${version}a${time} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/dataset:${version}a${time} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Create and Push Docker Release Manifest - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/coordinator:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/coordinator:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/coordinator:${{ steps.tag.outputs.TAG }}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/analytical:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/analytical:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/analytical:${{ steps.tag.outputs.TAG }}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:${{ steps.tag.outputs.TAG }}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:${{ steps.tag.outputs.TAG }}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }}-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }} \ - ${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }}-x86_64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/coordinator:latest \ - ${{ env.REGISTRY }}/graphscope/coordinator:latest-x86_64 \ - ${{ env.REGISTRY }}/graphscope/coordinator:latest-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/analytical:latest \ - ${{ env.REGISTRY }}/graphscope/analytical:latest-x86_64 \ - ${{ env.REGISTRY }}/graphscope/analytical:latest-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:latest \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:latest-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-frontend:latest-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:latest \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:latest-x86_64 \ - ${{ env.REGISTRY }}/graphscope/interactive-executor:latest-aarch64 - - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/learning:latest \ - ${{ env.REGISTRY }}/graphscope/learning:latest-x86_64 \ - ${{ env.REGISTRY }}/graphscope/learning:latest-aarch64 - - # push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/coordinator:${{ steps.tag.outputs.TAG }} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/analytical:${{ steps.tag.outputs.TAG }} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:${{ steps.tag.outputs.TAG }} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:${{ steps.tag.outputs.TAG }} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:${{ steps.tag.outputs.TAG }} - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/dataset:${{ steps.tag.outputs.TAG }} - - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/coordinator:latest - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/analytical:latest - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-frontend:latest - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/interactive-executor:latest - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/learning:latest - diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml deleted file mode 100644 index adc32da74139..000000000000 --- a/.github/workflows/build-graphscope-manylinux-ext-images.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Build GraphScope Manylinux-Ext Images - -# build graphscope-dev-base image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. -on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - 'python/graphscope/gsctl/**' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-manylinux-ext-image-x86-64: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux:ext-x86_64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - - build-manylinux-ext-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - if: false - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux2014:ext-aarch64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # clean - sudo docker rmi -f graphscope/manylinux2014:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} || true - - manifest_push_manylinux-ext-image: - runs-on: ubuntu-20.04 - if: false - needs: [build-manylinux-ext-image-x86-64, build-manylinux-ext-image-aarch64] - steps: - - name: Create and Push Docker Manifest - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-x86_64 \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-aarch64 - # push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml deleted file mode 100644 index 2f13331bc8de..000000000000 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ /dev/null @@ -1,359 +0,0 @@ -name: Build GraphScope Wheels on Linux - -# on: [push, pull_request] -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - push: - tags: - - "v*" - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-wheels-for-arm64: - # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - if: false - runs-on: [self-hosted, Linux, ARM64] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 - - - name: Build Wheel Package - run: | - # change the version for nightly release - # 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then - echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; - fi - - cd ${GITHUB_WORKSPACE}/k8s/internal - # build graphscope wheels - sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install - - # build client wheels - sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install - - # package - cd ${GITHUB_WORKSPACE} - sudo chown $(id -u) -R . - tar -zcf client.tar.gz python/dist/wheelhouse/*.whl - tar -zcf graphscope.tar.gz coordinator/dist/ - - # move wheels into one folder to upload to PyPI - mkdir ${GITHUB_WORKSPACE}/upload_pypi - mv ${GITHUB_WORKSPACE}/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - mv ${GITHUB_WORKSPACE}/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - # only wheel packages that platform aware need to be published. - # the wheel packages under the coordinator/dist directory are same to the amd64 platform - # mv ${GITHUB_WORKSPACE}/coordinator/dist/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: wheel-arm64-${{ github.sha }} - path: | - client.tar.gz - graphscope.tar.gz - retention-days: 5 - - # We do this, since failures on test.pypi aren't that bad - - name: Publish to Test PyPI - # the limit of graphscope package in test.pypi is still 100MB, which is not enough. - if: false - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_PASSWORD }} - repository_url: https://test.pypi.org/legacy/ - packages_dir: upload_pypi/ - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.2 - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} - packages_dir: upload_pypi/ - - build-wheels: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 - - - name: Build Wheel Package - run: | - bash ${GITHUB_WORKSPACE}/.github/free-disk-space.sh || true - - # Due to an observation of changing hostname in github runners, - # append 127.0.0.1 to etc/hosts to avoid DNS lookup. - r=`cat /etc/hosts | grep $(hostname) || true` - if [[ -z "${r}" ]]; then - export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; - fi - cat /etc/hosts - - # change the version for nightly release - # 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then - echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; - fi - - cd ${GITHUB_WORKSPACE}/k8s/internal - # build graphscope wheels - sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install - - # build client wheels - sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install - - # package - cd ${GITHUB_WORKSPACE} - sudo chown $(id -u) -R . - tar -zcf client.tar.gz python/dist/wheelhouse/*.whl - tar -zcf graphscope.tar.gz coordinator/dist/ - - # move wheels into one folder to upload to PyPI - mkdir ${GITHUB_WORKSPACE}/upload_pypi - mv ${GITHUB_WORKSPACE}/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - mv ${GITHUB_WORKSPACE}/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - mv ${GITHUB_WORKSPACE}/coordinator/dist/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: wheel-${{ github.sha }} - path: | - client.tar.gz - graphscope.tar.gz - retention-days: 5 - - # We do this, since failures on test.pypi aren't that bad - - name: Publish to Test PyPI - # the limit of graphscope package in test.pypi is still 100MB, which is not enough. - if: false - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_PASSWORD }} - repository_url: https://test.pypi.org/legacy/ - packages_dir: upload_pypi/ - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.2 - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} - packages_dir: upload_pypi/ - - build-image: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - runs-on: ubuntu-20.04 - needs: [build-wheels] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build GraphScope Image - run: | - cd ${GITHUB_WORKSPACE}/artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - cd ${GITHUB_WORKSPACE}/k8s/internal - make graphscope - - # make jupyter-image - cd ${GITHUB_WORKSPACE} - docker build --build-arg CI=${CI} -t graphscope/jupyter:${SHORT_SHA} -f ./k8s/internal/jupyter.Dockerfile . - - - name: Release Nightly Image - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}" - # jupyter image - sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # graphscope image - tag=${{ steps.tag.outputs.TAG }} - sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - - tag=latest - sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/jupyter:${tag} - - ubuntu-python-test: - if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }} - runs-on: ubuntu-20.04 - needs: [build-wheels] - strategy: - fail-fast: false - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Prepare Environment - shell: bash - run: | - echo "Python version: ${{ matrix.python-version }}" - # install graphscope-client - cd artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install $f --user || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install ./*.whl --user - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install ./*.whl --user - popd - - # install tensorflow - python3 -m pip install pytest "tensorflow" "pandas" --user - - # force upgrade protobuf and grpcio (may be downgraded by tensorflow) - python3 -m pip install -U protobuf grpcio --user - - # install java - sudo apt update -y && sudo apt install openjdk-11-jdk -y - - - name: Run Minimum Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - python3 -c "import sys; print(sys.version)" - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest - - centos-test: - if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }} - runs-on: ubuntu-20.04 - needs: [build-wheels] - container: - image: centos:8 - options: - --shm-size 4096m - - steps: - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Prepare Environment - shell: bash - run: | - # fixed download metadata for repo 'appstream' - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* - - dnf install -y gcc gcc-c++ make cmake - - # install git and python3 - dnf install -y git python38 python38-devel - python3 -m pip install --upgrade pip --user - - # install graphscope-client - cd ${GITHUB_WORKSPACE}/artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install $f --user || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install ./*.whl --user - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install ./*.whl --user || true - popd - - # install tensorflow - python3 -m pip install pytest "tensorflow" "pandas" --user - python3 -m pip install -U wrapt --user - - # force upgrade protobuf and grpcio (may be downgraded by tensorflow) - python3 -m pip install -U protobuf grpcio --user - - # install jdk - dnf install -y java-11-openjdk-devel - - - name: Run Minimum Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # python3-Cython version is too old, use cython installed by pip in $HOME/.local/bin - export PATH=$HOME/.local/bin:$PATH - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml deleted file mode 100644 index 418e51c628a2..000000000000 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ /dev/null @@ -1,327 +0,0 @@ -name: Build GraphScope Wheels on macOS - -# on: [push, pull_request] -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - push: - tags: - - "v*" - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-wheels: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - runs-on: macos-12 - strategy: - matrix: - python-version: ['3.11'] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Dependencies - run: | - brew update || true - brew install bash coreutils - - alias bash=$(brew --prefix)/bin/bash - - export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - sudo mkdir /opt/graphscope - sudo chown -R $(id -u):$(id -g) /opt/graphscope - - # Not sure when the first installation would fail, saying that the lock of python3.10 is taken - # However the second trial would success. - python3 -m pip install click - python3 gsctl.py install-deps dev || true - python3 gsctl.py install-deps dev - echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env - - - name: Install JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' # See 'Supported distributions' for available options - java-version: '11' - - - name: Reinstall glog - run: | - # Essential cause glog installed by brew is broken and produce corrupted messages - brew uninstall --ignore-dependencies glog - . ~/.graphscope_env - git clone https://github.com/google/glog.git -b v0.6.0 - cd glog - - cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF -DBUILD_TESTING=OFF - make -j - sudo make install - - # make delocate-wheel happy - sudo mkdir -p /usr/local/opt/glog/lib - sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/ - - - name: Install latest libgrape-lite - if: false - run: | - . ~/.graphscope_env - git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local - make -j4 - sudo make install - - - name: Install latest vineyard - if: false - run: | - source ~/.graphscope_env - python3 -m pip install libclang - git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j4 - sudo make install - - - name: Build Server Wheel - run: | - # source environment variable - . ~/.graphscope_env - python3 -m pip install libclang - echo ${CC} - - # make sure the python's local bin is in PATH (by `pip install --user`) - export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - # change the version for nightly release - # e.g. 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; - then - echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; - fi - - python3 -m pip install numpy pandas "grpcio>=1.49" "grpcio-tools>=1.49" "mypy-protobuf>=3.4.0" wheel - - # build graphscope server wheel - cd ${GITHUB_WORKSPACE}/k8s/internal - sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local - - # package - cd ${GITHUB_WORKSPACE} - tar -zcf graphscope.tar.gz coordinator/dist/wheelhouse/*.whl - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: wheel-macos-${{ github.sha }} - path: graphscope.tar.gz - retention-days: 5 - - build-client-wheels: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - runs-on: macos-12 - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Dependencies - run: | - brew update || true - brew install bash coreutils - alias bash=$(brew --prefix)/bin/bash - sudo mkdir /opt/graphscope - sudo chown -R $(id -u):$(id -g) /opt/graphscope - - python3 -m pip install click - python3 gsctl.py install-deps dev || true - python3 gsctl.py install-deps dev - - - name: Install latest libgrape-lite - if: false - run: | - . ~/.graphscope_env - git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local - make -j4 - sudo make install - - - name: Install latest vineyard - if: false - run: | - . ~/.graphscope_env - python3 -m pip install libclang - git clone --single-branch -b v0.20.3 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j4 - sudo make install - - - name: Build Client Wheels - run: | - python3 -c "import sys; print(sys.version)" - . ~/.graphscope_env - - # make sure the python's local bin is in PATH (by `pip install --user`) - export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - # change the version for nightly release - # 0.15.0 -> 0.15.0a20220808 - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then - echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; - fi - - # build graphscope client wheel - cd ${GITHUB_WORKSPACE}/k8s/internal - make graphscope-client-py3-package GRAPHSCOPE_HOME=/usr/local - - # package - cd ${GITHUB_WORKSPACE} - tar -zcf client.tar.gz python/dist/ - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: wheel-macos-${{ github.sha }}-${{ matrix.python-version }} - path: client.tar.gz - retention-days: 5 - - # Action gh-action-pypi-publish not support non-linux os. - publish-wheels: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - runs-on: ubuntu-20.04 - needs: [build-wheels, build-client-wheels] - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Prepare Wheels - env: - PYTHON: ${{ matrix.python-version }} - run: | - # move wheels into one folder to upload to PyPI - mkdir ${GITHUB_WORKSPACE}/upload_pypi - - cd ${GITHUB_WORKSPACE}/artifacts - tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz - mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - - if [ "$PYTHON" == "3.8" ]; then - tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz - mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - fi - - # We do this, since failures on test.pypi aren't that bad - - name: Publish to Test PyPI - # the limit of graphscope package in test.pypi is still 100MB, which is not enough. - if: false - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.TEST_PYPI_PASSWORD }} - repository_url: https://test.pypi.org/legacy/ - packages_dir: upload_pypi/ - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} - packages_dir: upload_pypi/ - - python-test: - if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }} - runs-on: macos-12 - needs: [build-wheels, build-client-wheels] - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Prepare Environment - run: | - cd ${GITHUB_WORKSPACE}/artifacts - # install graphscope-client - tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install $f --user || true; done - popd - # install graphscope - tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist/wheelhouse - python3 -m pip install ./*.whl --user - popd - - # install tensorflow - python3 -m pip install pytest "tensorflow" "pandas" --user - - - name: Run Minimum Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - export JAVA_HOME=$(/usr/libexec/java_home -v11) - export PATH=$JAVA_HOME/bin:$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - python3 -c "import sys; print(sys.version)" - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index c125bdc4a1ee..000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,266 +0,0 @@ -name: Docs - -on: - workflow_dispatch: - push: - branches: - - main - - docs - - dev/docs - tags: - - "v*" - pull_request: - branches: - - main - - docs - - dev/docs - -jobs: - build: - runs-on: ubuntu-20.04 - permissions: - issues: write - pull-requests: write - contents: write - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - submodules: true - fetch-depth: 0 - - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - src: - - 'docs/**' - - - name: Setup Java11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - - - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Leave a marker - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' }} - run: | - touch ${GITHUB_WORKSPACE}/preview-the-docs.mark - - - name: Leave the comment on pull request when started - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - ⚡️ Deploying PR Preview ${{ github.event.pull_request.head.sha }} to [surge.sh](https://alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh) ... [Build logs](https://github.com/alibaba/GraphScope/runs/${{ github.run_id }}) - - - - - - 🤖 By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Cpp Format and Lint Check - run: | - # install clang-format - sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format - sudo chmod +x /usr/bin/clang-format - - # run format - cd analytical_engine/ - find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.h" | xargs clang-format -i --style=file - find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.cc" | xargs clang-format -i --style=file - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo "| make gsa_clformat" - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with clang-format-8, which can be obtained from" - echo "|" - echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Java Format and Lint Check - run: | - wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar - - files_to_format=$(git ls-files *.java) - - # run formatter in-place - java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| google-java-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" - echo "|" - echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Python Format and Lint Check - run: | - echo "Checking formatting for $GITHUB_REPOSITORY" - pip3 install -r coordinator/requirements-dev.txt - pip3 install -r python/requirements-dev.txt - pushd python - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd - pushd coordinator - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - - - name: Generate Docs - shell: bash - run: | - # Install pip dependencies, build builtin gar, and generate proto stuffs. - sudo apt update - sudo apt install -y doxygen graphviz - - # generate a tagged version - cd ${GITHUB_WORKSPACE} - make graphscope-docs - - # generate a stable version - tag=$(git describe --exact-match --tags HEAD 2>/dev/null || true) - if [ ! -z "$tag" ]; - then - export TAG_VER=stable - make graphscope-docs - fi - - - name: Preview using surge - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - run: | - npm install -g surge - surge ./docs/_build/latest/html \ - alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh \ - --token ${{ secrets.SURGE_TOKEN }} - - - name: Leave the comment on pull request when succeed - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - 🎊 PR Preview ${{ github.event.pull_request.head.sha }} has been successfully built and deployed to https://alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh - - - - - - 🤖 By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Check file existence - id: check_files - if: ${{ failure() }} - uses: andstor/file-existence-action@v2 - with: - files: "preview-the-docs.mark" - - - name: Leave the comment on pull request when failed - if: ${{ failure() && steps.check_files.outputs.files_exists == 'true' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - 😭 Deploy PR Preview ${{ github.event.pull_request.head.sha }} failed. [Build logs](https://github.com/alibaba/GraphScope/runs/${{ github.run_id }}) - - - - - - 🤖 By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: false - - - name: Upload Docs - if: ${{ github.event_name == 'push' && github.repository == 'alibaba/GraphScope' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) }} - shell: bash - run: | - shopt -s extglob - rm google-java-format-1.13.0-all-deps.jar* || true - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - tag=$(git describe --exact-match --tags HEAD 2>/dev/null || true) - - git fetch origin gh-pages --no-recurse-submodules - git checkout gh-pages - - cd docs/ - - if [ -d "_build/latest" ]; - then - rm -rf latest - cp -R _build/latest/html ./latest - else - rm -rf latest - cp -R _build/${tag}/html ./latest - - rm -rf ${tag} - cp -R _build/${tag}/html ./${tag} - fi - - if [ -d "_build/stable" ]; - then - rm -rf !(_build|latest|stable|v*) - cp -R _build/stable/html/* ./ - fi - - rm -rf _build/ - rm -rf ../flex/ || true - rm -rf ../learning_engine/ || true - rm -rf ../python || true - rm -rf ../analytical_engine/java || true - - git add -A - git commit --amend --no-edit -m "Generate latest docs on CI, from commit ${{ github.sha }}." --author "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" -s --date="$(date -R)" - git push -f origin gh-pages - diff --git a/.github/workflows/flex-dummy.yml b/.github/workflows/flex-dummy.yml deleted file mode 100644 index 00cc8316101d..000000000000 --- a/.github/workflows/flex-dummy.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: GraphScope Flex CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!flex/**' - - '!.github/workflows/flex.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' \ No newline at end of file diff --git a/.github/workflows/flex.yml b/.github/workflows/flex.yml deleted file mode 100644 index 2cfd15d31bb2..000000000000 --- a/.github/workflows/flex.yml +++ /dev/null @@ -1,204 +0,0 @@ -name: GraphScope Flex CI - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths: - - 'flex/**' - - '.github/workflows/flex.yml' - pull_request: - branches: - - main - paths: - - 'flex/**' - - '.github/workflows/flex.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - test-build-flex: - runs-on: ubuntu-latest - - strategy: - matrix: - os: [ubuntu-20.04, ubuntu-22.04] - platform: [x86_64] - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Install dependencies - run: | - cd ${GITHUB_WORKSPACE}/flex - bash ./scripts/install_dependencies.sh 4 - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 - - - name: Build - run: | - cd ${GITHUB_WORKSPACE}/flex - git submodule update --init - mkdir build && cd build # only test default build - cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_DOC=OFF && sudo make -j 4 - - # test the different combination of cmake options: -DBUILD_HQPS=ON/OFF -DBUILD_TEST=ON/OFF, -DBUILD_ODPS_FRAGMENT_LOADER=ON/OFF - test-cmake-options: - runs-on: ubuntu-20.04 - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 - strategy: - matrix: - BUILD_HQPS: [ON, OFF] - BUILD_TEST: [ON, OFF] - BUILD_ODPS_FRAGMENT_LOADER: [ON, OFF] - steps: - - uses: actions/checkout@v3 - - - name: Build - run: | - cd ${GITHUB_WORKSPACE}/flex - git submodule update --init - mkdir build && cd build - cmake .. -DBUILD_HQPS=${{ matrix.BUILD_HQPS }} -DBUILD_TEST=${{ matrix.BUILD_TEST }} \ - -DBUILD_ODPS_FRAGMENT_LOADER=${{ matrix.BUILD_ODPS_FRAGMENT_LOADER }} - sudo make -j4 - - test-AOCC-compilation: - runs-on: ubuntu-20.04 - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 - steps: - - uses: actions/checkout@v3 - - - name: Download aocc compiler - run: | - cd ${GITHUB_WORKSPACE}/flex - wget https://download.amd.com/developer/eula/aocc/aocc-4-1/aocc-compiler-4.1.0_1_amd64.deb - sudo apt-get update && sudo apt-get install libncurses-dev libncurses5-dev - sudo dpkg -i aocc-compiler-4.1.0_1_amd64.deb - - - name: Test AOCC compiler - env: - CC: clang - CXX: clang++ - OMPI_CC: clang - OMPI_CXX: clang++ - run: | - . /opt/AMD/aocc-compiler-4.1.0/setenv_AOCC.sh - cd ${GITHUB_WORKSPACE}/flex - git submodule update --init - mkdir build && cd build - cmake .. -DBUILD_DOC=OFF -DCMAKE_BUILD_TYPE=DEBUG - sudo make -j4 - - test-flex: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 - steps: - - uses: actions/checkout@v3 - - - name: Install latest libgrape-lite - if: false - run: | - git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - mkdir -p build && cd build - cmake .. - make -j$(nproc) - make install - - - name: Build - env: - HOME: /home/graphscope/ - run: | - cd ${GITHUB_WORKSPACE}/flex - git submodule update --init - mkdir build && cd build - cmake .. && sudo make -j$(nproc) - - - name: Test GRIN on mutable csr - run: | - git submodule update --init - cd flex/engines/graph_db/grin - mkdir build && cd build - cmake .. && sudo make -j$(nproc) - export FLEX_DATA_DIR=../../../../interactive/examples/modern_graph/ - ${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/modern_graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/ - rm -r ./data/wal - rm -r ./data/runtime/* - ./run_grin_test 'flex://schema_file=../../../../interactive/examples/modern_graph/modern_graph.yaml&data_dir=./data/' - - - name: Prepare test dataset - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest/ - run: | - git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR} - - - name: Test String edge property on modern graph - env: - FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/ - run: | - rm -rf /tmp/csr-data-dir/ - cd ${GITHUB_WORKSPACE}/flex/build/ - SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_string_edge.yaml - BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml - GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ - GLOG_v=10 ./tests/rt_mutable_graph/string_edge_property_test ${SCHEMA_FILE} /tmp/csr-data-dir/ - - - name: Test build empty graph - run: | - rm -rf /tmp/csr-data-dir/ - cd ${GITHUB_WORKSPACE}/flex/build/ - GLOG_v=10 ./tests/rt_mutable_graph/test_empty_graph /tmp/csr-data-dir/ - - name: Test ACID - run: | - rm -rf /tmp/csr-data-dir/ - cd ${GITHUB_WORKSPACE}/flex/build/ - GLOG_v=10 ./tests/rt_mutable_graph/test_acid 8 /tmp/csr-data-dir/ - - - name: Test Graph Loading on modern graph - env: - FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/ - run: | - rm -rf /tmp/csr-data-dir/ - cd ${GITHUB_WORKSPACE}/flex/build/ - SCHEMA_FILE=../interactive/examples/modern_graph/modern_graph.yaml - BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml - GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ - - - name: Test Graph Loading on type_test graph - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest/ - FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/type_test/ - run: | - # remove modern graph indices - rm -rf /tmp/csr-data-dir/ - - cd ${GITHUB_WORKSPACE}/flex/build/ - SCHEMA_FILE=${GS_TEST_DIR}/type_test/graph.yaml - BULK_LOAD_FILE=${GS_TEST_DIR}/type_test/import.yaml - GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ -p 2 - - - name: Test Graph Loading on LDBC SNB sf0.1 - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest/ - FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/ldbc-sf01-long-date/ - run: | - # remove previous graph indices - rm -rf /tmp/csr-data-dir/ - - cd ${GITHUB_WORKSPACE}/flex/build/ - SCHEMA_FILE=${FLEX_DATA_DIR}/audit_graph_schema.yaml - BULK_LOAD_FILE=${FLEX_DATA_DIR}/audit_bulk_load.yaml - GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ -p 2 diff --git a/.github/workflows/gae-dummy.yml b/.github/workflows/gae-dummy.yml deleted file mode 100644 index b405f337efc3..000000000000 --- a/.github/workflows/gae-dummy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: GraphScope GAE CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!analytical_engine/**' - - '!.github/workflows/gae.yml' - - '!coordinator/gscoordinator/template/CMakeLists.template' # to verify whether java app can be successfully built when CMakeLists.template changes. - - 'analytical_engine/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml deleted file mode 100644 index 7fc093bce577..000000000000 --- a/.github/workflows/gae.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: GraphScope GAE CI - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths: - - 'analytical_engine/**' - - '.github/workflows/gae.yml' - - 'coordinator/gscoordinator/template/CMakeLists.template' # to verify whether java app can be successfully built when CMakeLists.template changes. - - '!analytical_engine/**.md' - pull_request: - branches: - - main - paths: - - 'analytical_engine/**' - - '.github/workflows/gae.yml' - - 'coordinator/gscoordinator/template/CMakeLists.template' # to verify whether java app can be successfully built when CMakeLists.template changes. - - '!analytical_engine/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.20.3 - steps: - - uses: actions/checkout@v3 - - - name: Install latest vineyard - if: false - run: | - . /home/graphscope/.graphscope_env - sudo pip3 install libclang - git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - mkdir -p build && cd build - cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \ - -DCMAKE_INSTALL_PREFIX=/opt/graphscope \ - -DOPENSSL_ROOT_DIR=/opt/graphscope \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j$(nproc) - sudo make install - - - name: Build - env: - RUN_JAVA_TESTS: ON - run: | - # default install to "/opt/graphscope" - make analytical-java BUILD_TEST=ON NUMPROC=1 - sudo make analytical-java-install - # also make coordinator and client for python test - python3 -m pip install --upgrade pip setuptools - make coordinator - - - name: Run Cpp Test - env: - GRAPHSCOPE_HOME: /opt/graphscope - GS_TEST_DIR: ${{ github.workspace }}/gstest - RUN_JAVA_TESTS: ON - run: | - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - # java maven test - cd ${GITHUB_WORKSPACE}/analytical_engine/java - mvn test -Dmaven.antrun.skip=true --quiet - - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar - # for giraph test - export GIRAPH_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-giraph/target/grape-giraph-${version}-shaded.jar - - cd ${GITHUB_WORKSPACE}/analytical_engine/build - ../test/app_tests.sh --test_dir ${GS_TEST_DIR} - - - name: Run Java Python Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - RUN_JAVA_TESTS: ON - GRAPHSCOPE_HOME: /opt/graphscope - run: | - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar - - cd ${GITHUB_WORKSPACE}/python - python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \ - --cov-report=term --exitfirst -s -v graphscope/tests/unittest/test_java_app.py - - - name: Upload Coverage - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - file: ./python/coverage.xml - fail_ci_if_error: false diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml deleted file mode 100644 index d028d9aa7d09..000000000000 --- a/.github/workflows/gaia.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: GraphScope GAIA CI - -on: - # Trigger the workflow on push or pull request, but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths: - - 'interactive_engine/**' - - '.github/workflows/gaia.yml' - pull_request: - branches: - - main - paths: - - 'interactive_engine/**' - - '.github/workflows/gaia.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - gaia-test: - runs-on: [self-hosted, manylinux2014] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Detect the tmate session - run: | - if grep -v "grep" .github/workflows/gaia.yml | grep "action-tmate"; then - echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' - exit 1 - fi - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Rust Format Check - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/executor && ./check_format.sh - - - name: Build Pegasus and Test - run: | - echo $(pwd) - cd ${GITHUB_WORKSPACE}/interactive_engine/executor/engine/pegasus - cargo build --verbose - cargo test --verbose - - - name: Build Ir on Experimental Store and Csr Store - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler - make build - make build rpc.target=start_rpc_server_csr - - - name: Ir Unit Test - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && make test - - - name: CommonType Unit Test - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/executor/common/dyn_type && cargo test - - - name: Store Unit Test - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/executor/store/exp_store && cargo test - cd ${GITHUB_WORKSPACE}/interactive_engine/executor/store/groot && cargo test - # TODO: add ut in global_query - # cd ${GITHUB_WORKSPACE}/interactive_engine/executor/store/global_query && cargo test - - - name: Ir Integration Test on Experimental Store - run: | - . ${HOME}/.graphscope_env - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git /tmp/gstest - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_experimental_ci.sh - - - name: Ir Integration Pattern Test on Experimental Store - run: | - cd ${GITHUB_WORKSPACE} - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_experimental_advanced_ci.sh - - - name: Ir Integration Test on Csr Store - run: | - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_csr_ci.sh - - - name: Ir Integration Pattern Test on Csr Store - run: | - cd ${GITHUB_WORKSPACE} - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_csr_pattern_ci.sh - - - name: Upload GIE log - if: failure() - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - name: gie-log - path: /var/log/graphscope diff --git a/.github/workflows/gss-dummy.yml b/.github/workflows/gss-dummy.yml deleted file mode 100644 index 41a96fb71907..000000000000 --- a/.github/workflows/gss-dummy.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: GraphScope Store CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!proto/**' - - '!interactive_engine/**' - - '!python/graphscope/client/**' - - '!charts/graphscope-store/**' - - '!.github/workflows/gss.yml' - - 'interactive_engine/**.md' - - 'charts/graphscope-store/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store - -jobs: - gremlin-test: - # Require the host is able to run docker without sudo and - # can `ssh localhost` without password, which may need to - # be configured manually when a new self-hosted runner is added. - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - helm-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [gremlin-test] - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml deleted file mode 100644 index 7bc4309b0b02..000000000000 --- a/.github/workflows/gss.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: GraphScope Store CI - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - 'CONTRIBUTORS' - - 'LICENSE' - - 'NOTICE.txt' - - '**.md' - - '**.rst' - - 'docs/**' - - 'demo/**' - - 'scripts/**' - - 'tutorials/**' - pull_request: - branches: - - main - paths: - - 'proto/**' - - 'interactive_engine/**' - - 'python/graphscope/client/**' - - 'charts/graphscope-store/**' - - '.github/workflows/gss.yml' - - '!interactive_engine/**.md' - - '!charts/graphscope-store/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store - -jobs: - gremlin-test: - # Require the host is able to run docker without sudo and - # can `ssh localhost` without password, which may need to - # be configured manually when a new self-hosted runner is added. - runs-on: [self-hosted, manylinux2014] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - name: Detect the tmate session - run: | - if grep -v "grep" .github/workflows/gss.yml | grep "action-tmate"; then - echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' - exit 1 - fi - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Build GraphScope Store - run: | - . ${HOME}/.graphscope_env - export SCCACHE_DIR=~/.cache/sccache - export RUSTC_WRAPPER=/usr/local/bin/sccache - cd ${GITHUB_WORKSPACE}/interactive_engine - mvn clean install -P groot -Drust.compile.mode=debug -DskipTests --quiet - - sccache --show-stats - - - name: Gremlin Test - run: | - . ${HOME}/.graphscope_env - cd interactive_engine/groot-server - mvn test -P gremlin-test - - - name: Upload tools for helm test to Artifact - uses: actions/upload-artifact@v3 - with: - name: groot - path: | - interactive_engine/assembly/target/groot.tar.gz - interactive_engine/data-load-tool/target/data-load-tool-0.0.1-SNAPSHOT.jar - retention-days: 5 - - helm-test: - runs-on: [self-hosted, ubuntu2004] - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [gremlin-test] - env: - JAVA_HOME: /usr/lib/jvm/default-java - GS_TEST_DIR: ${{ github.workspace }}/gstest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Detect the tmate session - run: | - if grep -v "grep" .github/workflows/gss.yml | grep "action-tmate"; then - echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' - exit 1 - fi - - - uses: actions/download-artifact@v3 - with: - name: groot - path: artifacts - - - name: Set GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Prepare Image - run: | - ls -la artifacts/*/* - mv artifacts/assembly/target/groot.tar.gz artifacts/ - mv artifacts/data-load-tool/target/data-load-tool-0.0.1-SNAPSHOT.jar artifacts/ - docker build -t ${{ env.GSS_IMAGE }}:${SHORT_SHA} \ - -f .github/workflows/docker/graphscope-store.Dockerfile . - - - name: Setup SSH - run: | - ssh-keygen -t rsa -f ~/.ssh/id_rsa -N '' - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - chmod og-wx ~/.ssh/authorized_keys - echo "StrictHostKeyChecking no" >> ~/.ssh/config - sudo /etc/init.d/ssh start - - - name: Create the kubernetes cluster - run: | - # download gstest - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \ - --cpus='12' --memory='32000mb' --disk-size='40000mb' - - minikube image load ${{ env.GSS_IMAGE }}:${SHORT_SHA} - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/helm-installation-logs - NAMESPACES: "gs*,default" - MODE: start - - - name: Install graphscope-store with helm - run: | - # update helm dependency - cd ${GITHUB_WORKSPACE}/charts/graphscope-store - helm dependency update - - # helm deployment and testing - cd ${GITHUB_WORKSPACE}/charts - helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store - helm test ci --timeout 5m0s - - - name: Test the helm deployment - run: | - # 1. get gss service endpoint - export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend) - export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend) - export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") - - # 2. deploy hadoop hdfs - tar -zxf /home/runner/hadoop-2.10.1.tar.gz -C /tmp/ - cd ${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts - ./prepare_hadoop.sh /tmp/hadoop-2.10.1 - export PATH=${PATH}:/tmp/hadoop-2.10.1/bin - - # data-load-tool is needed for offline_load.sh - # see .github/workflows/hadoop_scripts/offline_load.sh - - export LOADER_DIR=${GITHUB_WORKSPACE}/artifacts - export LOAD_DATA_SCRIPT=${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts/offline_load.sh - sed s/GRAPH_ENDPOINT/$NODE_IP:$GRPC_PORT/ databuild.config.template > databuild.config - - # 3. upload data to HDFS - hadoop fs -mkdir /ldbc_sample || true - hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv - hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_knows_person_0_0.csv /ldbc_sample/person_knows_person_0_0.csv - # python test - cd ${GITHUB_WORKSPACE}/python - python3 -m pip install -r ./requirements.txt --user - python3 -m pip install -r ./requirements-dev.txt --user - python3 -m pip install pytest-cov --user - python3 setup.py build_proto - python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_fresh - - - name: restart helm and run demo with the PersistentVolume - run: | - # restart helm and run demo with the PersistentVolume - helm uninstall ci - sleep 30 - cd ${GITHUB_WORKSPACE}/charts - helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store - - - name: Helm Test with Helm Deployment and PersistentVolume - run: | - # helm test and python test on the restarted store - export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend) - export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend) - export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") - helm test ci --timeout 10m0s - - - name: Python Test with Helm Deployment and PersistentVolume - run: | - export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend) - export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend) - export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") - cd ${GITHUB_WORKSPACE}/python - python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_after_restart - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - OUTPUT_DIR: ${{ github.workspace }}/helm-installation-logs - MODE: stop - - - name: upload the k8s logs to artifact - if: ${{ failure() }} - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - name: k8s-test-logs - path: ${{ github.workspace }}/helm-installation-logs - diff --git a/.github/workflows/hqps-db-ci.yml b/.github/workflows/hqps-db-ci.yml deleted file mode 100644 index 1f1d2760399a..000000000000 --- a/.github/workflows/hqps-db-ci.yml +++ /dev/null @@ -1,223 +0,0 @@ -name: GraphScope GIE HighQPS DB CI - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths: - - 'flex/**' - - 'interactive_engine/**' - - '.github/workflows/hqps-db-ci.yml' - pull_request: - branches: - - main - paths: - - 'flex/**' - - 'interactive_engine/**' - - '.github/workflows/hqps-db-ci.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - test-hqps-engine: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9 - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install latest libgrape-lite - if: false - run: | - git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - mkdir -p build && cd build - cmake .. - make -j$(nproc) - make install - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v3 - - - name: Build - env: - GIE_HOME: ${{ github.workspace }}/interactive_engine/ - HOME: /home/graphscope/ - run: | - cd ${GITHUB_WORKSPACE}/ - git submodule update --init - cd ${GITHUB_WORKSPACE}/flex - mkdir build && cd build - cmake .. && sudo make -j$(nproc) - sudo make install - - # cargo - . /home/graphscope/.cargo/env - which cargo - - # build compiler - cd ${GIE_HOME}/ - mvn clean install -Pexperimental -DskipTests - - - name: Prepare dataset and workspace - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - INTERACTIVE_WORKSPACE: /tmp/interactive_workspace - run: | - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR} - mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc - GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml - BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml - cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml - cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml - mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies - cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml - cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml - mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo - cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml - cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml - - # load graph - cd ${GITHUB_WORKSPACE}/flex/build - export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date - GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/ - export FLEX_DATA_DIR=../interactive/examples/movies - GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/ - export FLEX_DATA_DIR=../interactive/examples/graph_algo - GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/ - - - name: Test HQPS admin http service - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph - run: - cd ${GITHUB_WORKSPACE}/flex/tests/hqps - bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR} - - - name: Sample Query test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - cd ${GITHUB_WORKSPACE}/flex/build - export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date - ./bin/bulk_loader -g ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \ - -l ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml -d /tmp/csr-data-dir/ - rm -r /tmp/csr-data-dir/runtime/* - rm -r /tmp/csr-data-dir/wal - ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \ - /tmp/csr-data-dir/ - - - name: Run codegen test. - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - HOME : /home/graphscope/ - INTERACTIVE_WORKSPACE: /tmp/interactive_workspace - run: | - cd ${GITHUB_WORKSPACE}/flex/bin - - for i in 1 2 3 4 5 6 7 8 9 10 11 12; - do - cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " - cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" - echo $cmd - eval ${cmd} - done - - for i in 1 2 3 4 5 6 7 8 9 10 11 12; - do - cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " - cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" - echo $cmd - eval ${cmd} - done - - # test movie graph, 8,9,10 are not supported now - # change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml - for i in 1 2 3 4 5 6 7 11 12 13 14 15; - do - cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/" - cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " - cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml" - echo $cmd - eval ${cmd} - done - - - name: Test cypher procedure generation - run: | - cd ${GITHUB_WORKSPACE}/flex/bin - ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \ - --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \ - --graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml - - ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \ - --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \ - --graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml - - - name: Run End-to-End cypher adhoc ldbc query test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - HOME : /home/graphscope/ - INTERACTIVE_WORKSPACE: /tmp/interactive_workspace - run: | - cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ - export ENGINE_TYPE=hiactor - # change the default_graph config in ./engine_config_test.yaml to ldbc - sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml - bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml - - - name: Run End-to-End cypher adhoc movie query test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - HOME : /home/graphscope/ - INTERACTIVE_WORKSPACE: /tmp/interactive_workspace - run: | - cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ - export ENGINE_TYPE=hiactor - # change the default_graph config in ./engine_config_test.yaml to movies - sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml - bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml - - - name: Run End-to-End cypher adhoc graph_algo query test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - HOME : /home/graphscope/ - INTERACTIVE_WORKSPACE: /tmp/interactive_workspace - run: | - cd ${GITHUB_WORKSPACE}/flex/tests/hqps/ - export ENGINE_TYPE=hiactor - # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo - sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml - bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \ - ${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml - - diff --git a/.github/workflows/k8s-ci-dummy.yml b/.github/workflows/k8s-ci-dummy.yml deleted file mode 100644 index f24c6e19351e..000000000000 --- a/.github/workflows/k8s-ci-dummy.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: GraphScope CI on Kubernetes (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/k8s-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v3 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-analytical: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-analytical-java: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-interactive: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-learning: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - # build-coordinator: - # runs-on: ubuntu-20.04 - # needs: [changes] - # if: ${{ github.repository == 'alibaba/GraphScope' }} - # steps: - # - run: 'echo "No action required" ' - - k8s-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - steps: - - run: 'echo "No action required" ' - - gie-test: - runs-on: ubuntu-20.04 - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - if: ${{ (needs.changes.outputs.gie-function-test == 'false' || github.ref == 'refs/heads/main') && github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml deleted file mode 100644 index 0ecb6803be56..000000000000 --- a/.github/workflows/k8s-ci.yml +++ /dev/null @@ -1,743 +0,0 @@ -name: GraphScope CI on Kubernetes - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - 'CONTRIBUTORS' - - 'LICENSE' - - 'NOTICE.txt' - - '**.md' - - '**.rst' - - 'docs/**' - - 'demo/**' - - 'scripts/**' - - 'tutorials/**' - pull_request: - branches: - - main - paths: - - '.github/workflows/k8s-ci.yml' - - 'Makefile' - - 'analytical_engine/**' - - 'charts/**' - - 'coordinator/**' - - 'interactive_engine/**' - - 'k8s/**' - - 'learning_engine/**' - - 'proto/**' - - 'python/**' - - '!**.md' - - '!**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v3 - - - name: Detect the tmate session - run: | - if grep -v "grep" .github/workflows/k8s-ci.yml | grep "action-tmate"; then - echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' - exit 1 - fi - - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - gie-k8s-failover-test: - - 'interactive_engine/**' - - 'charts/gie-standalone/**' - - - name: Cpp Format and Lint Check - run: | - # install clang-format - sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format - sudo chmod +x /usr/bin/clang-format - - # collect the source files - cd analytical_engine/ - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - # run format - clang-format -i --style=file $(echo $files) - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo "| make gsa_clformat" - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with clang-format-8, which can be obtained from" - echo "|" - echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - # validate cpplint - function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; } - - # run cpplint - ec ./misc/cpplint.py $(echo $files) - if [[ "$ec" != "0" ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| cpplint failures found! Run: " - echo "|" - echo "| make gsa_cpplint" - echo "|" - echo "| to fix this error." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Python Format and Lint Check - run: | - echo "Checking formatting for $GITHUB_REPOSITORY" - pip3 install -r coordinator/requirements-dev.txt - pushd python - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd - pushd coordinator - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - - - name: Setup Java11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - - - name: Java Format and Lint Check - run: | - wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar - - files_to_format=$(git ls-files *.java) - - # run formatter in-place - java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| google-java-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" - echo "|" - echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - build-analytical: - runs-on: [self-hosted, manylinux2014] - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - name: Build Artifact - run: | - . ${HOME}/.graphscope_env - export INSTALL_DIR=${{ github.workspace }}/install - mkdir ${INSTALL_DIR} - make analytical-install INSTALL_PREFIX=${INSTALL_DIR} - strip ${INSTALL_DIR}/bin/grape_engine - strip ${INSTALL_DIR}/lib/*.so - sudo cp -rs ${INSTALL_DIR}/* ${GRAPHSCOPE_HOME}/ - python3 ./k8s/utils/precompile.py --graph --output_dir ${INSTALL_DIR}/builtin - strip ${INSTALL_DIR}/builtin/*/*.so - - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: analytical - path: ${{ github.workspace }}/install - retention-days: 5 - - build-analytical-java: - runs-on: [self-hosted, manylinux2014] - needs: [changes] - #if: ${{ github.repository == 'alibaba/GraphScope' }} - if: false - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Build Artifact - run: | - . ${HOME}/.graphscope_env - export INSTALL_DIR=${{ github.workspace }}/install - mkdir ${INSTALL_DIR} - make analytical-java-install INSTALL_PREFIX=${INSTALL_DIR} - strip ${INSTALL_DIR}/bin/grape_engine - strip ${INSTALL_DIR}/bin/graphx_runner - strip ${INSTALL_DIR}/lib/*.so - sudo cp -rs ${INSTALL_DIR}/* ${GRAPHSCOPE_HOME}/ - python3 ./k8s/utils/precompile.py --graph --output_dir ${INSTALL_DIR}/builtin - strip ${INSTALL_DIR}/builtin/*/*.so - - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: analytical-java - path: ${{ github.workspace }}/install - retention-days: 5 - - build-interactive: - runs-on: [self-hosted, manylinux2014] - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Build Artifact - run: | - . ${HOME}/.graphscope_env - mkdir install - export SCCACHE_DIR=~/.cache/sccache - export RUSTC_WRAPPER=/usr/local/bin/sccache - sccache --start-server - make interactive-install BUILD_TYPE="debug" INSTALL_PREFIX=${{ github.workspace }}/install - strip install/bin/gaia_executor - sccache --show-stats - - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: interactive - path: ${{ github.workspace }}/install - retention-days: 5 - - build-learning: - runs-on: [self-hosted, manylinux2014] - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Build Artifact - run: | - mkdir install - make learning-install INSTALL_PREFIX=${{ github.workspace }}/install - cd ${{ github.workspace }}/python - pip3 install -r requirements.txt -r requirements-dev.txt --user - python3 setup.py bdist_wheel - auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") - sed -i 's/p.error/logger.warning/g' ${auditwheel_path} - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ github.workspace }}/learning_engine/graph-learn/graphlearn/built/lib - auditwheel repair dist/*.whl - python3 -m pip install wheelhouse/*.whl; \ - cp wheelhouse/*.whl ${{ github.workspace }}/install/ - cd ${{ github.workspace }}/coordinator - pip3 install -r requirements.txt -r requirements-dev.txt --user - python3 setup.py bdist_wheel - cp dist/*.whl ${{ github.workspace }}/install/ - - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: learning - path: ${{ github.workspace }}/install - retention-days: 5 - - # build-coordinator: - # runs-on: ubuntu-latest - # needs: [changes] - # if: false # Wheels of learning includes coordinator and client - # steps: - # - uses: actions/checkout@v3g - - k8s-test: - runs-on: [self-hosted, ubuntu2004] - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-analytical, build-interactive, build-learning] - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build Images - run: | - cd ${GITHUB_WORKSPACE}/k8s - # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder - make analytical CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - make interactive CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - make coordinator CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - - docker images - - - name: Install Python dependencies - run: | - cd ${GITHUB_WORKSPACE}/python - pip3 install -r requirements.txt -r requirements-dev.txt - pip3 install pytest pytest-cov pytest-timeout pytest-xdist - - # build python client proto - cd ${GITHUB_WORKSPACE}/python - python3 setup.py build_proto - - # install mars - # python3 -m pip install pymars==0.8.0 - - - name: Setup SSH - run: | - ssh-keygen -t rsa -f ~/.ssh/id_rsa -N '' - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - chmod og-wx ~/.ssh/authorized_keys - echo "StrictHostKeyChecking no" >> ~/.ssh/config - sudo /etc/init.d/ssh start - - - name: Prepare Kubernetes - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \ - --cpus='12' --memory='32000mb' --disk-size='40000mb' \ - --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}" - mkdir -p ${{ github.workspace }} - - - name: Load Images - run: | - # create a kubernetes cluster with 2 nodes - #TODO(caoye) - #minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.32' \ - # --nodes 2 --cpus='6' --memory='16000mb' --disk-size='20000mb' \ - # --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}" - - export GS_REGISTRY="" - export GS_TAG=${SHORT_SHA} - minikube image load graphscope/coordinator:${SHORT_SHA} - echo "Loaded coordinator" - minikube image load graphscope/analytical:${SHORT_SHA} - echo "Loaded analytical" - minikube image load graphscope/interactive-frontend:${SHORT_SHA} - echo "Loaded frontend" - minikube image load graphscope/interactive-executor:${SHORT_SHA} - echo "Loaded executor" - # minikube image load graphscope/learning:${SHORT_SHA} - # echo "loaded learning" - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-helm-installation-logs - NAMESPACES: "gs*,default" - MODE: start - - - name: Helm Test of installation - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - cd charts - helm template graphscope --set image.registry="",image.tag="${SHORT_SHA}",withJupyter=false,volumes.enabled=true,volumes.items.data.field.path=${GS_TEST_DIR},engines.log_level=DEBUG \ - ./graphscope - helm install graphscope --set image.registry="",image.tag="${SHORT_SHA}",withJupyter=false,volumes.enabled=true,volumes.items.data.field.path=${GS_TEST_DIR},engines.log_level=DEBUG \ - ./graphscope - helm test graphscope --timeout 5m0s - - - name: Test demo script of helm installation - run: | - export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}") - export NODE_PORT=$(kubectl get services coordinator-service-graphscope -ojsonpath="{.spec.ports[0].nodePort}") - echo "GraphScope service listen on ${NODE_IP}:${NODE_PORT}" - export GS_ADDR=${NODE_IP}:${NODE_PORT} - cd ${GITHUB_WORKSPACE}/python - - python3 -m pytest -d --tx popen//python=python3 \ - -s -vvv \ - ./graphscope/tests/kubernetes/test_demo_script.py \ - -k test_helm_installation - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-helm-installation-logs - MODE: stop - - - name: Delete Helm Cluster - run: helm delete graphscope - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-test-logs - NAMESPACES: "gs*,default" - MODE: start - - - name: Kubernetes Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - export GS_REGISTRY="" - export GS_TAG=${SHORT_SHA} - - cd ${GITHUB_WORKSPACE}/python - export PATH=${HOME}/.local/bin:${PATH} - - python3 -m pytest -d --tx popen//python=python3 \ - --ignore=./graphscope/tests/kubernetes/test_store_service.py \ - --cov=graphscope --cov-config=.coveragerc --cov-report=xml --cov-report=term \ - -s -vvv --log-cli-level=INFO \ - ./graphscope/tests/kubernetes - - - name: Upload Coverage - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - file: ./python/coverage.xml - fail_ci_if_error: false - - - name: HDFS test - env: - JAVA_HOME: /usr/lib/jvm/default-java - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - export GS_REGISTRY="" - export GS_TAG=${SHORT_SHA} - - # install hadoop HDFS - tar -zxf /home/runner/hadoop-2.10.1.tar.gz -C /tmp/ - cd ${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts - ./prepare_hadoop.sh /tmp/hadoop-2.10.1 - export PATH=${PATH}:/tmp/hadoop-2.10.1/bin - - # upload data to HDFS - hadoop fs -mkdir /ldbc_sample || true - hadoop fs -chmod 777 /ldbc_sample - hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv - hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_knows_person_0_0.csv /ldbc_sample/person_knows_person_0_0.csv - - # prepare CI environments - export HDFS_TEST_DIR=hdfs:///ldbc_sample - export HDFS_HOST=$(hostname -I | awk '{print $1}') - - # run test - cd ${GITHUB_WORKSPACE}/python - python3 -m pytest -d --tx popen//python=python3 \ - -s -vvv \ - ./graphscope/tests/kubernetes/test_demo_script.py \ - -k test_demo_on_hdfs - # Check the result file have successfully written to the given location - # hdfs dfs -test -e /ldbc_sample/res.csv_0 && hdfs dfs -test -e /ldbc_sample/res.csv_1 - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - OUTPUT_DIR: ${{ github.workspace }}/k8s-ci-demo-script-test-logs - MODE: stop - - - name: Upload the k8s logs to artifact - uses: actions/upload-artifact@v3 - if: ${{ always() }} - continue-on-error: true - with: - name: k8s-test-logs - path: ${{ github.workspace }}/k8s-ci-*-logs - - gie-test: - runs-on: [self-hosted, ubuntu2004] - needs: [build-analytical, build-interactive, build-learning] - if: ${{ needs.changes.outputs.gie-function-test == 'true' && github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build Images - run: | - cd ${GITHUB_WORKSPACE}/k8s - # Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space to pull the builder - make analytical CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - make interactive CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - make coordinator CI=true VERSION=${SHORT_SHA} REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com BUILDER_VERSION=ci - - docker images - - - name: Install Python dependencies - run: | - cd ${GITHUB_WORKSPACE}/python - pip3 install -r requirements.txt -r requirements-dev.txt - pip3 install pytest pytest-cov pytest-timeout pytest-xdist - - # build python client proto - python3 setup.py build_proto - - - name: Run Function Test - run: | - export GS_TEST_DIR=${GITHUB_WORKSPACE}/interactive_engine/tests/src/main/resources - minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \ - --cpus='12' --memory='32000mb' --disk-size='60000mb' \ - --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}" - - export GS_REGISTRY="" - export GS_TAG=${SHORT_SHA} - minikube image load graphscope/coordinator:${SHORT_SHA} - minikube image load graphscope/analytical:${SHORT_SHA} - minikube image load graphscope/interactive-frontend:${SHORT_SHA} - minikube image load graphscope/interactive-executor:${SHORT_SHA} - - export PYTHONPATH=${GITHUB_WORKSPACE}/python:${PYTHONPATH} - cd ${GITHUB_WORKSPACE}/interactive_engine && mvn clean install --quiet -DskipTests -Drust.compile.skip=true -P graphscope - cd ${GITHUB_WORKSPACE}/interactive_engine/tests - # ./function_test.sh 8111 1 - ./function_test.sh 8112 2 - - build-gie-experimental: - needs: [changes] - # Require the user id of the self-hosted is 1001, which may need to be - # configured manually when a new self-hosted runner is added. - runs-on: [self-hosted, manylinux2014] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Build GIE Experimental Artifacts - run: | - . ~/.graphscope_env - export RPC_TARGET=start_rpc_server_k8s - cd interactive_engine/compiler && make build rpc.target=${RPC_TARGET} - cd ${GITHUB_WORKSPACE} - strip interactive_engine/executor/ir/target/release/${RPC_TARGET} - strip interactive_engine/executor/ir/target/release/libir_core.so - tar -czf artifacts.tar.gz interactive_engine/compiler/target/libs \ - interactive_engine/compiler/target/compiler-0.0.1-SNAPSHOT.jar \ - interactive_engine/compiler/conf \ - interactive_engine/compiler/set_properties.sh \ - interactive_engine/executor/ir/target/release/libir_core.so \ - interactive_engine/executor/ir/target/release/${RPC_TARGET} - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: gie-experimental - path: | - artifacts.tar.gz - retention-days: 5 - - gie-k8s-failover-test: - needs: [build-gie-experimental] - runs-on: [self-hosted, ubuntu2004] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/download-artifact@v3 - with: - name: gie-experimental - path: artifacts - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Prepare Docker Image - run: | - docker build \ - -t registry.cn-hongkong.aliyuncs.com/graphscope/interactive-experimental:${SHORT_SHA} \ - -f .github/workflows/docker/interactive-experimental-local-artifacts.Dockerfile . - - - name: Prepare Cluster and Data - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - STORE_DATA_PATH: /tmp/data - GIE_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/interactive-experimental - run: | - # prepare graph data - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - mkdir -p ${STORE_DATA_PATH} - cp -r ${GS_TEST_DIR}/modern_graph_exp_bin/* ${STORE_DATA_PATH} - - # prepare minikube cluster - minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \ - --cpus='12' --memory='32000mb' --disk-size='60000mb' \ - --mount=true --mount-string="${STORE_DATA_PATH}:${STORE_DATA_PATH}" - minikube image load ${GIE_IMAGE}:${SHORT_SHA} - - # install python gremlin client - pip install gremlinpython - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - SHOW_TIMESTAMPS: 'true' - OUTPUT_DIR: ${{ github.workspace }}/k8s-failover-logs - NAMESPACES: "gs*,default" - MODE: start - - - name: Run K8S Failover Test - run: | - cd ${GITHUB_WORKSPACE}/charts - # create local persistent volume which contains graph data for test - kubectl apply -f ./gie-standalone/tools/pvc.yaml - # create gie instance (compiler & executor & exp storage) - helm install test ./gie-standalone \ - --set frontend.image.repository=graphscope/interactive-experimental \ - --set frontend.image.tag=${SHORT_SHA} \ - --set executor.image.repository=graphscope/interactive-experimental \ - --set executor.image.tag=${SHORT_SHA} \ - --set storageType=Experimental \ - --set schemaConfig=expr_modern_schema.json \ - --set executor.replicaCount=2 \ - --set frontend.service.type=NodePort - # run failover test - cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_k8s_failover_ci.sh default test-gie-standalone 2 1 - - - uses: dashanji/kubernetes-log-export-action@v5 - env: - OUTPUT_DIR: ${{ github.workspace }}/k8s-failover-logs - MODE: stop - - - name: upload the k8s logs to artifact - uses: actions/upload-artifact@v3 - continue-on-error: true - if: ${{ always() }} - with: - name: k8s-failover-logs - path: ${{ github.workspace }}/k8s-failover-logs diff --git a/.github/workflows/local-ci-dummy.yml b/.github/workflows/local-ci-dummy.yml deleted file mode 100644 index 9282d85fc6cc..000000000000 --- a/.github/workflows/local-ci-dummy.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: GraphScope CI on Local (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/local-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v3 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-wheels: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - mini-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-wheels] - steps: - - run: 'echo "No action required" ' - - python-unittest: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-basic-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-algo-and-generator-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml deleted file mode 100644 index 4fd81085350d..000000000000 --- a/.github/workflows/local-ci.yml +++ /dev/null @@ -1,577 +0,0 @@ -name: GraphScope CI on Local - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - push: - branches: - - main - paths-ignore: - - 'CONTRIBUTORS' - - 'LICENSE' - - 'NOTICE.txt' - - '**.md' - - '**.rst' - - 'docs/**' - - 'demo/**' - - 'scripts/**' - - 'tutorials/**' - pull_request: - branches: - - main - paths: - - '.github/workflows/local-ci.yml' - - 'Makefile' - - 'analytical_engine/**' - - 'charts/**' - - 'coordinator/**' - - 'interactive_engine/**' - - 'k8s/**' - - 'learning_engine/**' - - 'proto/**' - - 'python/**' - - '!**.md' - - '!**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v3 - - - name: Detect the tmate session - run: | - # hack here: get all field of build-wheels which run on self-hosted machine - if sed -n '/self-hosted/,/mini-test/ p' .github/workflows/local-ci.yml | \ - grep -v "grep" - | grep "action-tmate"; then - echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' - exit 1 - fi - - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - gie-k8s-failover-test: - - 'interactive_engine/**' - - 'charts/ir-standalone/**' - - - name: Cpp Format and Lint Check - run: | - # install clang-format - sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format - sudo chmod +x /usr/bin/clang-format - - # collect the source files - cd analytical_engine/ - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - # run format - clang-format -i --style=file $(echo $files) - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo "| make gsa_clformat" - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with clang-format-8, which can be obtained from" - echo "|" - echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - # validate cpplint - function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; } - - # run cpplint - ec ./misc/cpplint.py $(echo $files) - if [[ "$ec" != "0" ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| cpplint failures found! Run: " - echo "|" - echo "| make gsa_cpplint" - echo "|" - echo "| to fix this error." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Python Format and Lint Check - run: | - echo "Checking formatting for $GITHUB_REPOSITORY" - pip3 install -r coordinator/requirements-dev.txt - pushd python - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd - pushd coordinator - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - - - name: Setup Java11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - - - name: Java Format and Lint Check - run: | - wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar - - files_to_format=$(git ls-files *.java) - - # run formatter in-place - java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| google-java-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" - echo "|" - echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - build-wheels: - # Require the user id of the self-hosted is 1001, which may need to be - # configured manually when a new self-hosted runner is added. - runs-on: [self-hosted, ubuntu2004] - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/.cache/sccache - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Build Wheels - run: | - #sleep infinity - git submodule update --init - cd ${GITHUB_WORKSPACE}/learning_engine/graph-learn && git submodule update --init third_party/pybind11 - - # build graphscope wheels - cd ${GITHUB_WORKSPACE}/k8s/internal - # sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope - make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com - - # build client wheels - # sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope - make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com - - # package - cd ${GITHUB_WORKSPACE} - sudo chown $(id -u) -R . - tar -zcf client.tar.gz python/dist/wheelhouse/*.whl - tar -zcf graphscope.tar.gz coordinator/dist/ - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: wheel-${{ github.sha }} - path: | - client.tar.gz - graphscope.tar.gz - retention-days: 5 - - mini-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-wheels] - steps: - - uses: actions/checkout@v3 - - - uses: actions/download-artifact@v3 - with: - path: artifacts - - - name: Prepare Environment - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # install graphscope-client - cd artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install --no-cache-dir $f || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install --no-cache-dir ./*.whl - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install --no-cache-dir ./*.whl - popd - - # install tensorflow - python3 -m pip install --no-cache-dir pytest pytest-xdist "tensorflow" "pandas" --user - # install pytorch - python3 -m pip install --no-cache-dir "torch" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu - # install java - sudo apt update -y && sudo apt install openjdk-11-jdk -y - - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: false - - - name: Run Minimum Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - python3 -m pytest -d --tx popen//python=python3 \ - -s -v \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest - - - name: Upload GIE log - if: failure() - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - name: gie-log - path: ~/.local/log/graphscope - - python-unittest: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - defaults: - run: - shell: bash --noprofile --norc -eo pipefail {0} - strategy: - matrix: - deployment: ["standalone", "distributed"] - - steps: - - uses: actions/checkout@v3 - if: ${{ needs.changes.outputs.gae-python == 'true' }} - - - uses: actions/download-artifact@v3 - if: ${{ needs.changes.outputs.gae-python == 'true' }} - with: - path: artifacts - - - name: Prepare Environment - if: ${{ needs.changes.outputs.gae-python == 'true' }} - run: | - # install graphscope-client - cd artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install --no-cache-dir $f || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install --no-cache-dir ./*.whl - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install --no-cache-dir ./*.whl - popd - - # install pytest - python3 -m pip install --no-cache-dir pytest pytest-cov pytest-timeout pytest-xdist - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: false - - - name: Run Python Test - if: ${{ needs.changes.outputs.gae-python == 'true' }} - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - DEPLOYMENT: ${{ matrix.deployment }} - run: | - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - python3 -m pytest -d --tx popen//python=python3 \ - -s -v \ - --cov=graphscope --cov-config=python/.coveragerc --cov-report=xml --cov-report=term \ - --exitfirst \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/unittest - - - name: Upload Coverage - if: ${{ needs.changes.outputs.gae-python == 'true' || github.ref == 'refs/heads/main' }} - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - file: ./coverage.xml - fail_ci_if_error: false - - networkx-basic-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - defaults: - run: - shell: bash - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - uses: actions/checkout@v3 - if: ${{ needs.changes.outputs.networkx == 'true' }} - - - uses: dorny/paths-filter@v2 - if: ${{ needs.changes.outputs.networkx == 'true' }} - id: nx-filter - with: - filters: | - basic: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/__init__.py' - - 'python/graphscope/nx/classes/**' - - 'python/graphscope/nx/!(tests)' - - 'python/graphscope/nx/tests/!(convert)' - convert: - - 'python/graphscope/nx/convert.py' - - 'python/graphscope/nx/convert_matrix.py' - - 'python/graphscope/nx/tests/convert/**' - - - uses: actions/download-artifact@v3 - if: ${{ needs.changes.outputs.networkx == 'true' }} - with: - path: artifacts - - - name: Prepare Environment - if: ${{ needs.changes.outputs.networkx == 'true' }} - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # install graphscope-client - cd artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install --no-cache-dir $f || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install --no-cache-dir ./*.whl - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install --no-cache-dir ./*.whl - popd - - # install pytest - python3 -m pip install --no-cache-dir pytest pytest-xdist - - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: false - - - name: Basic Test - if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.basic == 'true' }} - env: - DEPLOYMENT: ${{ matrix.deployment }} - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - pip3 show networkx - python3 -m pytest -d --tx popen//python=python3 \ - --exitfirst -s -vvv \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests \ - --ignore=$(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert - - - name: Convert Test - if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.convert == 'true' }} - env: - DEPLOYMENT: ${{ matrix.deployment }} - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - pip3 show networkx - python3 -m pytest -d --tx popen//python=python3 \ - --exitfirst -s -vvv \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert - - networkx-algo-and-generator-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - uses: actions/checkout@v3 - if: ${{ needs.changes.outputs.networkx == 'true' }} - - - uses: dorny/paths-filter@v2 - if: ${{ needs.changes.outputs.networkx == 'true' }} - id: nx-filter - with: - filters: | - algorithm: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'python/graphscope/nx/algorithms/builtin.py' - - 'python/graphscope/nx/algorithms/tests/builtin/**' - generator: - - 'python/graphscope/nx/classes/**' - - 'python/graphscope/nx/utils/**' - - 'python/graphscope/nx/generators/**' - io: - - 'python/graphscope/nx/readwrite/**' - - - uses: actions/download-artifact@v3 - if: ${{ needs.changes.outputs.networkx == 'true' }} - with: - path: artifacts - - - name: Prepare Environment - if: ${{ needs.changes.outputs.networkx == 'true' }} - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # install graphscope-client - cd artifacts - tar -zxf ./wheel-${{ github.sha }}/client.tar.gz - pushd python/dist/wheelhouse - for f in * ; do python3 -m pip install --no-cache-dir $f || true; done - popd - - # install graphscope - tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz - pushd coordinator/dist - python3 -m pip install --no-cache-dir ./*.whl - popd - pushd coordinator/dist/wheelhouse - python3 -m pip install --no-cache-dir ./*.whl - popd - - # install pytest - python3 -m pip install --no-cache-dir pytest pytest-xdist - - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - - - name: Algorithm test - if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.algorithm == 'true' }} - env: - DEPLOYMENT: ${{ matrix.deployment }} - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - pip3 show networkx - python3 -m pytest -d --tx popen//python=python3 \ - --exitfirst -s -v \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/algorithms/tests/builtin - - - name: Generator test - if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.generator == 'true' }} - env: - DEPLOYMENT: ${{ matrix.deployment }} - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - pip3 show networkx - python3 -m pytest -d --tx popen//python=python3 \ - --exitfirst -s -v \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/generators/tests - - - name: Readwrite test - if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.io == 'true' }} - env: - DEPLOYMENT: ${{ matrix.deployment }} - run: | - python3 -m pytest -d --tx popen//python=python3 \ - --exitfirst -s -v -m "not slow" \ - $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/readwrite/tests diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml deleted file mode 100644 index 2ad78324239c..000000000000 --- a/.github/workflows/networkx-forward-algo-nightly.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: NetworkX Forward algorithm CI - -# on: [push, pull_request] -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - -jobs: - forward-algo-test: - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - runs-on: ubuntu-20.04 - defaults: - run: - shell: bash --noprofile --norc -eo pipefail {0} - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.20.3 - options: - --shm-size 4096m - - steps: - - uses: actions/checkout@v3 - - - name: Build GAE and coordinator - run: | - pushd ${GITHUB_WORKSPACE} - . /home/graphscope/.graphscope_env - install_prefix=/opt/graphscope - sudo mkdir -p ${install_prefix} - sudo make analytical-install BUILD_TEST=OFF NUMPROC=1 INSTALL_PREFIX=${install_prefix} - # also make coordinator and client for python test - make coordinator && make client - popd - - - name: Run Forward algorithms test - env: - DEPLOYMENT: 'standalone' - run: | - cd ${GITHUB_WORKSPACE}/python - . /home/graphscope/.graphscope_env - export GRAPHSCOPE_HOME=/opt/graphscope - python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \ - --cov-report=term -s -v -m "not slow" graphscope/nx/algorithms/tests/forward - - - name: Upload Coverage - uses: codecov/codecov-action@v3 - continue-on-error: true - with: - file: ./python/coverage.xml - fail_ci_if_error: false diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 3cd0671b87c5..000000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: GraphScope Nightly Build CI - -# on: [push, pull_request] -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - -jobs: - nightly-test: - if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash --noprofile --norc -eo pipefail {0} - strategy: - matrix: - os: [ubuntu-22.04, macos-latest] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Install Dependencies of GraphScope - run: | - # Due to an observation of changing hostname in github runners, - # append 127.0.0.1 to etc/hosts to avoid DNS lookup. - r=`cat /etc/hosts | grep $(hostname) || true` - if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi - cat /etc/hosts - - if [ "$RUNNER_OS" == "macOS" ]; then - brew update - # pushd $(brew --repo homebrew/core) - # # fixed openmpi version 4.1.4 - # git checkout d2d8d2251eb3ce044993a79e5ad816ed3356bc36 - # popd - brew install bash - alias bash=$(brew --prefix)/bin/bash - fi - sudo mkdir /opt/graphscope - sudo chown -R $(id -u):$(id -g) /opt/graphscope - python3 -m pip install click - python3 gsctl.py install-deps dev --v6d-version v0.20.3 - - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v2 - - - name: Build GraphScope - run: | - . ${HOME}/.graphscope_env - # fixes nightly CI - export PATH=/Users/runner/Library/Python/3.11/bin:$PATH - make install INSTALL_PREFIX=/opt/graphscope - - - name: Run Python Test - if: matrix.os == 'ubuntu-20.04' - env: - FULL_TEST_SUITE: ON - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - # download dataset - git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} - # run test - . ${HOME}/.graphscope_env - make unittest - - - name: Upload Coverage - uses: codecov/codecov-action@v3 - if: matrix.os == 'ubuntu-20.04' - continue-on-error: true - with: - file: ./python/coverage.xml - fail_ci_if_error: false diff --git a/.github/workflows/pegasus.yml b/.github/workflows/pegasus.yml deleted file mode 100644 index ff4f204c2c9c..000000000000 --- a/.github/workflows/pegasus.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: GraphScope Pegasus CI - -on: - workflow_dispatch: - push: - branches: - - gaia - paths: - - 'interactive_engine/executor/engine/pegasus/**' - - '.github/workflows/pegasus.yml' - - pull_request: - branches: - - gaia - paths: - - 'interactive_engine/executor/engine/pegasus/**' - - '.github/workflows/pegasus.yml' - -env: - CARGO_TERM_COLOR: always - -jobs: - pegasus-build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - name: Build & Test - run: | - echo $(pwd) - cd interactive_engine/executor/engine/pegasus - cargo build --verbose - cargo test --verbose - - - name: Run example - run: | - cd research/engine/pegasus - - cargo build --release --examples - - # Run word_count_toy - target/release/examples/word_count_toy - - target/release/examples/word_count_toy -w 2 - - target/release/examples/word_count_toy -w 4 - - # Run logistic regression - target/release/examples/logistic_regression --data pegasus/examples/data/binary.csv - -# k8s-test: -# runs-on: [self-hosted, ubuntu2004] -# steps: -# - uses: actions/checkout@v3 -# -# - name: Detect the tmate session -# run: | -# if grep -v "grep" .github/workflows/pegasus.yml | grep "action-tmate"; then -# echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' -# exit 1 -# fi -# -# - name : Prepare Image -# run: | -# docker pull registry-vpc.cn-hongkong.aliyuncs.com/graphscope/pegasus-base:latest -# docker tag registry-vpc.cn-hongkong.aliyuncs.com/graphscope/pegasus-base:latest \ -# registry.cn-hongkong.aliyuncs.com/graphscope/pegasus-base:latest -# docker build -t registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }} \ -# --network=host \ -# -f research/engine/pegasus/test/k8s/pegasus.Dockerfile . -# echo " - name: CONFIG_MAP_NAME" >> research/engine/pegasus/test/k8s/pegasus-set.yaml -# echo " value: \"${{ github.sha }}\"" >> research/engine/pegasus/test/k8s/pegasus-set.yaml -# echo " image: registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }}" >> research/engine/pegasus/test/k8s/pegasus-set.yaml -# -# - name: Start k8s cluster -# run: | -# minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \ -# --cpus='12' --memory='32000mb' --disk-size='40000mb' -# minikube image load registry.cn-hongkong.aliyuncs.com/graphscope/pegasus:${{ github.sha }} -# -# kubectl get ns pegasus-ci || kubectl create ns pegasus-ci -# kubectl --namespace pegasus-ci create -f research/engine/pegasus/test/k8s/role_binding.yaml -# kubectl --namespace pegasus-ci create -f research/engine/pegasus/test/k8s/pegasus-set.yaml -# bash research/engine/pegasus/test/k8s/read_result.sh ${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 723110d9b32c..000000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: Cut Release - -# on: [push, pull_request] - -on: - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 03:00(CST) every day. - - cron: '00 19 * * *' - push: - tags: - - 'v*' - -env: - DOCKER_URL: registry.cn-hongkong.aliyuncs.com - GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store - HELM_CHARTS_URL: https://graphscope.oss-cn-beijing.aliyuncs.com/charts/ - OSS_REGION: oss-cn-beijing - OSS_ENDPOINT: oss-cn-beijing.aliyuncs.com - -jobs: - build-gss-image-x86-64: - if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build GraphScope Store Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - sudo make graphscope-store VERSION=${SHORT_SHA} - - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # x86_64 - # arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}" - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # x86_64 - # arch=$(uname -m) - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} - sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} - - build-gss-image-aarch64: - # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - if: false - runs-on: [self-hosted, Linux, ARM64] - steps: - - uses: actions/checkout@v3 - - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - - name: Build GraphScope Store Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - sudo make graphscope-store VERSION=${SHORT_SHA} - - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Release Image - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # aarch64 - arch=$(uname -m) - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - - - name: Clean Nightly Image - if: false - run: | - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - # clean - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${tag} || true - - - name: Clean Release Image - if: always() - run: | - # aarch64 - arch=$(uname -m) - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} || true - - push-gss-image-manifest: - # if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - if: false - runs-on: ubuntu-20.04 - - needs: [build-gss-image-x86-64, build-gss-image-aarch64] - - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Extract Tag Name - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - - # create - sudo docker manifest create \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-aarch64 - - # push - sudo docker manifest push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} - - release-helm-charts: - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - # this step is needed for update index.yaml - - name: Download Released Charts - env: - oss_accesskey_id: ${{ secrets.ALIYUN_OSS_ACCESSKEY_ID }} - oss_accesskey_secret: ${{ secrets.ALIYUN_OSS_ACCESSKEY_SECRET }} - run: | - # download ossutils - sudo wget -O /usr/bin/ossutil64 http://gosspublic.alicdn.com/ossutil/1.7.7/ossutil64 - sudo chmod +x /usr/bin/ossutil64 - - # download released charts - ossutil64 -i "${oss_accesskey_id}" \ - -k "${oss_accesskey_secret}" \ - -e ${{ env.OSS_ENDPOINT }} \ - cp -r oss://graphscope/charts ./charts - - # ossutil64 will downloads contents to folder ./charts/charts - pushd charts/ - mv charts/*.tgz charts/*.yaml ./ - popd - - - name: Package Charts - run: | - # download helm - if ! command -v helm &> /dev/null; then - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash - fi - - cd ${GITHUB_WORKSPACE}/charts - # package graphscope - helm package ./graphscope - # package graphscope-store - helm package --dependency-update ./graphscope-store - # package gie-standalone - helm package --dependency-update ./gie-standalone - # generate index.yaml - helm repo index . --url ${{ env.HELM_CHARTS_URL }} - - - name: Extract Tag Name - id: tag - run: echo "TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - - name: Upload Charts to OSS - uses: tvrcgo/upload-to-oss@master - with: - key-id: ${{ secrets.ALIYUN_OSS_ACCESSKEY_ID }} - key-secret: ${{ secrets.ALIYUN_OSS_ACCESSKEY_SECRET }} - region: ${{ env.OSS_REGION }} - bucket: graphscope - assets: | - charts/graphscope-${{ steps.tag.outputs.TAG }}.tgz:/charts/graphscope-${{ steps.tag.outputs.TAG }}.tgz - charts/graphscope-store-${{ steps.tag.outputs.TAG }}.tgz:/charts/graphscope-store-${{ steps.tag.outputs.TAG }}.tgz - charts/gie-standalone-${{ steps.tag.outputs.TAG }}.tgz:/charts/gie-standalone-${{ steps.tag.outputs.TAG }}.tgz - charts/index.yaml:/charts/index.yaml diff --git a/.github/workflows/spell-check.yml b/.github/workflows/spell-check.yml new file mode 100644 index 000000000000..656286bb0be2 --- /dev/null +++ b/.github/workflows/spell-check.yml @@ -0,0 +1,11 @@ +name: 'Check spelling' +on: # rebuild any PRs and main branch changes + pull_request: + push: + +jobs: + spellcheck: # run the action + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: streetsidesoftware/cspell-action@v5 \ No newline at end of file diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 64603d392ee1..000000000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: 'Check stale issues and pull requests' - -on: - schedule: - - cron: '0 0 12 * *' - workflow_dispatch: - -env: - DEFAULT_ASSIGNEE: yecol sighingnow - DAYS_BEFORE_ISSUE_CLOSE: -1 - DAYS_BEFORE_PR_CLOSE: -1 - DAYS_BEFORE_ISSUE_STALE: 7 - DAYS_BEFORE_PR_STALE: 14 - -jobs: - stale: - permissions: - issues: write - pull-requests: write - runs-on: ubuntu-latest - steps: - - name: Install dependencies - run: | - # install jq - sudo apt-get update - sudo apt-get install -y jq - - # install gh cli - type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y) - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ - && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && sudo apt update \ - && sudo apt install gh -y - - - uses: actions/stale@v9 - id: stale - with: - days-before-issue-close: ${{ env.DAYS_BEFORE_ISSUE_CLOSE }} - days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} - days-before-issue-stale: ${{ env.DAYS_BEFORE_ISSUE_STALE }} - days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }} - stale-issue-label: stale - stale-pr-label: stale - exempt-pr-labels: work-in-progress,requires-further-info,requires-further-discussion - exempt-issue-labels: work-in-progress,requires-further-info,requires-further-discussion,wontfix,newcomers - labels-to-remove-when-unstale: stale,requires-further-info,requires-further-discussion - exempt-draft-pr: true - - - name: Notify assignees - env: - REPO: ${{ github.repository }} - STALED: ${{ steps.stale.outputs.staled-issues-prs }} - GH_TOKEN: ${{ github.token }} - run: | - for N in $(echo "$STALED" | jq -r ".[].number"); - do - assignees=$(echo $(gh issue view $N -R $REPO --json assignees) | jq -r ".assignees[].login") - echo "Processing stale issue/pr $N, assignees: $assignees" - - message="" - if [ -z "$assignees" ]; then - message="/cc" - for assignee in $DEFAULT_ASSIGNEE; do - message="$message @$assignee" - done - message="$message, this issus/pr has had no activity for a long time, please help to review the status and assign people to work on it." - else - message="/cc" - for assignee in $assignees; do - message="$message @$assignee" - done - message="$message, this issus/pr has had no activity for for a long time, could you folks help to review the status ? \n" - message="$message To suppress further notifications,\n" - message="$message - for issues, \n" - message="$message - if it is waiting for further response from the reporter/author, please help to add the label \`requires-further-info\`, \n" - message="$message - if you have already started working on it, please add the label \`work-in-progress\` to the issue, \n" - message="$message - if this issue requires further designing discussion and not in current plan, or won't be fixed, please add the label \`requires-further-discussion\` or \`wontfix\` to the issue, \n" - message="$message - for pull requests, \n" - message="$message - if you are still working on it and it is not ready for reviewing, please convert this pull request as draft PR, \n" - message="$message - if you have decided to hold this development on, please add the \`requires-further-discussion\` label to the pull request. \n" - message="$message Thanks!" - fi - - echo -e "$message" > message-$N - message_content=$(cat message-$N) - echo "Commenting on stale issue/pr $N, assignees: $assignees, message: $message_content" - gh issue comment $N -R $REPO -b "$message_content" - done diff --git a/learning_engine/graph-learn b/learning_engine/graph-learn index 9682c9b399a5..a3fb2f1f1a90 160000 --- a/learning_engine/graph-learn +++ b/learning_engine/graph-learn @@ -1 +1 @@ -Subproject commit 9682c9b399a57ea318325cdb0c9f4f2f79644e82 +Subproject commit a3fb2f1f1a90e29ccb724f1547502b9dca7d14d0