diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..359e06718 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,794 @@ +name: Build Taichi +on: + workflow_call: + inputs: + build_id: + required: true + type: string + project_name: + required: false + default: 'taichi' + type: string + secrets: + workflow_dispatch: + inputs: + build_id: + required: true + type: string + project_name: + required: false + default: 'taichi' + type: string + secrets: + +concurrency: + group: build-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +env: + TI_CI: "1" + TI_SKIP_VERSION_CHECK: 'ON' + CI_IMAGE_VERSION: '202304251731' + TI_USE_GIT_CACHE: ${{ vars.TI_USE_GIT_CACHE }} + +jobs: + show_environ: + name: Show Environment Variables + # Disable this workflow on forks + if: github.repository_owner == 'taichi-dev' + runs-on: [self-hosted, Linux] + steps: + - name: Environment Variables + run: env + - name: Github Object + run: | + cat <<'EOF' + ${{ toJson(github) }} + EOF + + build_cpu_mac: + name: Build macos (CPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 120 || 180 }} + strategy: + matrix: + include: + - os: macos-10.15 + python: 3.7 + with_cpp_tests: ON + wanted_archs: 'cpu,vulkan' + runs-on: + - self-hosted + - ${{ matrix.os }} + env: + PY: ${{ matrix.python }} + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=ON + -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + env: + PLATFORM: 'macos' + + - name: Build & Install + if: needs.check_files.outputs.run_job == 'true' + run: | + # Use the molten-vk v1.1.10 downloaded from taichi assets + brew uninstall molten-vk -f + .github/workflows/scripts/build.py + env: + CXX: clang++ + + + build_gpu_linux: + name: Build (GPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + strategy: + matrix: + extra_markers: + - sm70 + - not sm70 + driver: + - driver470 + - driver510 + + runs-on: + - self-hosted + - cn + - cuda + - vulkan + - ${{ matrix.driver }} + - ${{ matrix.extra_markers == 'sm70' && 'sm70' || 'Linux' }} + env: + PY: '3.11' + PROJECT_NAME: taichi + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_BACKTRACE:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + TI_WANTED_ARCHS: 'cpu,cuda,vulkan,opengl,gles' + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '1' + + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + env: + EXTRA_TEST_MARKERS: ${{ matrix.extra_markers }} + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + build_manylinux2014: + name: Build (manylinux2014) + needs: check_files + runs-on: [self-hosted, cuda, cn, release] + env: + PY: "3.8" + PROJECT_NAME: taichi + TI_WANTED_ARCHS: "cuda,cpu" + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=OFF + -DTI_BUILD_TESTS:BOOL=ON + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-build-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + + build_amdgpu_linux: + name: Build (AMDGPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + runs-on: [self-hosted, amdgpu] + env: + PY: '3.8' + PROJECT_NAME: taichi + TI_WANTED_ARCHS: 'cpu,amdgpu' + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '0' + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_CUDA:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=OFF + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_AMDGPU:BOOL=ON + + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-amdgpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-amdgpu:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-amdgpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-amdgpu:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + + build_windows: + name: Build Windows + needs: check_files + strategy: + matrix: + extra_markers: + - sm70 + - not sm70 + runs-on: + - self-hosted + - cn + - windows + - cuda + - OpenGL + - ${{ matrix.extra_markers == 'sm70' && 'sm70' || 'windows' }} + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 180 }} + env: + PY: "3.7" + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_DX11:BOOL=ON + -DTI_WITH_DX12:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_BACKTRACE=ON + -DTI_WITH_C_API=ON + TI_WANTED_ARCHS: cpu,cuda,vulkan,opengl,gles + TI_SKIP_VERSION_CHECK: ON + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '1' + + steps: + - name: Workaround checkout Needed single revision issue + run: | + git config --system core.longpaths true + git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' # no '|| true' here + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + shell: cmd + run: | + python .\.github\workflows\scripts\build.py + + - name: Test + id: test + shell: pwsh + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + .\.github\workflows\scripts\win_test.ps1 -libsDir "$env:LocalAppData/buildbot" + env: + EXTRA_TEST_MARKERS: ${{ matrix.extra_markers }} + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Cleanup Git Cache Configs + shell: pwsh + if: always() + run: | + . .github/workflows/scripts/common-utils.ps1 + UnsetGitCachingProxy + exit 0 + + build_m1: + name: Build (Apple M1) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 60 || 120 }} + strategy: + matrix: + include: + - os: macos-latest + python: 3.8 + defaults: + run: + # https://github.com/actions/runner/issues/805#issuecomment-844426478 + shell: '/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}' + runs-on: [self-hosted, m1] + env: + CXX: clang++ + PY: ${{ matrix.python }} + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_CUDA:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_C_API=ON + -DTI_WITH_STATIC_C_API=ON + TI_WANTED_ARCHS: 'cpu,metal,vulkan' + PLATFORM: 'm1' + TI_RUN_RELEASE_TESTS: '1' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + brew install molten-vk + brew install llvm@15 + .github/workflows/scripts/build.py + env: + CXX: clang++ + + - name: Check C-API Export Symbols + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + + .github/workflows/scripts/aot-demo.sh check-c-api-export-symbols + + - name: Test + id: test + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + .github/workflows/scripts/unix_test.sh + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + + build_ios_capi: + name: Build iOS C-API Static Library + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 60 || 120 }} + runs-on: [self-hosted, m1] + env: + PY: "3.10" + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: .github/workflows/scripts/build.py ios + + - name: Save Compiled Static Library + if: ${{ needs.check_files.outputs.run_job != 'false' }} + uses: actions/upload-artifact@v3 + with: + name: libtaichi_c_api.iOS.a + path: 'dist/C-API-iOS/*.a' + retention-days: 7 + + aot_build_android_apps: + name: AOT Build Android Apps + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API:BOOL=ON + + - name: Test For Android AOT + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run-gpu --name taichi-test-android \ + registry.botmaster.tgr/taichi-test-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-smoke-test-android-aot-demo + + env: + TI_DEVICE_MEMORY_GB: '0.1' + + aot_build_android_headless_demos: + name: AOT Build Android Headless Demos + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API=ON + + - name: Build & Run C-API Headless Demos (Android) + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run-gpu --name taichi-test-capi-headless-demo \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-test-headless-demo + env: + TI_DEVICE_MEMORY_GB: '0.1' + + aot_build_android_unity_examples: + name: AOT Build Android Unity Examples + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API=ON + + - name: Prepare Unity Build Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-prepare-unity-build-env \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh prepare-unity-build-env + + - name: Build Taichi-UnityExample + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-unity-demo \ + registry.taichigraphics.com/unityci-editor:ubuntu-2020.3.14f1-android-1-with-secret-sauce \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-unity-demo + + - name: Run Taichi-UnityExample (C-API) + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-run-unity-demo \ + registry.botmaster.tgr/taichi-test-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh smoke-test-unity-demo + + aot_build_gpu_linux: + name: AOT Build (GPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + # TODO(proton): do we need to test both drivers? + strategy: + matrix: + tags: + - [self-hosted, cuda, vulkan, cn, driver470] + - [self-hosted, cuda, vulkan, cn, driver510] + env: + PY: '3.8' + PROJECT_NAME: taichi + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_BACKTRACE:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_C_API=ON + + runs-on: ${{ matrix.tags }} + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Check C-API Export Symbols + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test-check-c-api-export-symbols \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh check-c-api-export-symbols + + - name: Build & Run C-API Headless Demos (Desktop) + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + chown -R 1000:1000 . + ci-docker-run-gpu --name taichi-test-capi-headless-demo-desktop \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-test-headless-demo-desktop + env: + TI_DEVICE_MEMORY_GB: '0.1' + + - name: Compatibility Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_aot_compat_test.sh diff --git a/.github/workflows/initiator.yaml b/.github/workflows/initiator.yaml new file mode 100644 index 000000000..d76e1de2a --- /dev/null +++ b/.github/workflows/initiator.yaml @@ -0,0 +1,964 @@ +name: Taichi Workflow Initiator +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - master + - rc-* + schedule: + - cron: '0 18 * * *' # For testing the offline cache, GMT 18 == GMT+8 02 + +concurrency: + group: ${{ github.event.number || github.run_id }} + cancel-in-progress: true + +env: + TI_CI: "1" + TI_SKIP_VERSION_CHECK: 'ON' + TI_LITE_TEST: ${{ github.event_name == 'pull_request' && ! contains(github.event.pull_request.labels.*.name, 'full-ci') && ! startsWith(github.base_ref, 'rc-') && '1' || '' }} + TI_TEST_OFFLINE_CACHE: ${{ github.event.schedule == '0 18 * * *' && '1' || '' }} + CI_IMAGE_VERSION: '202304251731' + TI_USE_GIT_CACHE: ${{ vars.TI_USE_GIT_CACHE }} + +jobs: + show_environ: + name: Show Environment Variables + # Disable this workflow on forks + if: github.repository_owner == 'taichi-dev' + runs-on: [self-hosted, Linux] + steps: + - name: Environment Variables + run: env + - name: Github Object + run: | + cat <<'EOF' + ${{ toJson(github) }} + EOF + + check_files: + name: Check files + # Disable this workflow on forks + if: github.repository_owner == 'taichi-dev' + outputs: + run_job: ${{ steps.check_files.outputs.run_job }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + - name: check modified files + id: check_files + run: | + echo "Concurrency group: ${{ github.event.number || github.run_id }}" + echo "=============== list modified files ===============" + git diff --name-only @^ + + chore_files=( LICENSE CONTRIBUTING.md README.md netlify.toml ) + chore_dirs=( docs ) + run_job=false + + for file in $(git diff --name-only @^); do + is_chore=false + + for chore_file in ${chore_files[*]}; do + [[ ${file} == ${chore_file} ]] && is_chore=true && break + done + + for chore_dir in ${chore_dirs[*]}; do + [[ ${file} == ${chore_dir}/* ]] && is_chore=true && break + done + + if ! ${is_chore}; then + run_job=true + break + fi + done + + if ${run_job}; then + echo "run_job=true" >> $GITHUB_OUTPUT + else + echo "run_job=false" >> $GITHUB_OUTPUT + fi + + check_static_analyzer: + name: Check Static Analyzer + runs-on: ubuntu-latest + needs: check_files + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: clang-tidy + run: | + if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then + exit 0 + fi + # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio + echo $CR_PAT | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker pull ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.3.3 + docker run -id --user dev --name check_clang_tidy ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.3.3 /bin/bash + tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - check_clang_tidy:/home/dev/ + docker exec --user root check_clang_tidy apt-get update + docker exec --user root check_clang_tidy apt install -y clang-tidy-10 libx11-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev zlib1g-dev libglew-dev libglfw3-dev + docker exec --user dev check_clang_tidy /home/dev/taichi/.github/workflows/scripts/check_clang_tidy.sh "$CI_SETUP_CMAKE_ARGS" + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + CI_SETUP_CMAKE_ARGS: -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DTI_WITH_BACKTRACE:BOOL=ON + + build_and_test_cpu_mac: + name: Build and Test macos (CPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 120 || 180 }} + strategy: + matrix: + include: + - os: macos-10.15 + python: 3.7 + with_cpp_tests: ON + wanted_archs: 'cpu,vulkan' + runs-on: + - self-hosted + - ${{ matrix.os }} + env: + PY: ${{ matrix.python }} + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=ON + -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + env: + PLATFORM: 'macos' + + - name: Build & Install + if: needs.check_files.outputs.run_job == 'true' + run: | + # Use the molten-vk v1.1.10 downloaded from taichi assets + brew uninstall molten-vk -f + .github/workflows/scripts/build.py + env: + CXX: clang++ + + # [DEBUG] Copy this step around to enable debugging inside Github Action instances. + #- name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true + + - name: Test + id: test + if: needs.check_files.outputs.run_job == 'true' + run: .github/workflows/scripts/unix_test.sh + env: + TI_WANTED_ARCHS: ${{ matrix.wanted_archs }} + TI_SKIP_CPP_TESTS: Disabled because Vulkan is supported but not working on buildbot4 + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + build_and_test_gpu_linux: + name: Build and Test (GPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + strategy: + matrix: + extra_markers: + - sm70 + - not sm70 + driver: + - driver470 + - driver510 + + runs-on: + - self-hosted + - cn + - cuda + - vulkan + - ${{ matrix.driver }} + - ${{ matrix.extra_markers == 'sm70' && 'sm70' || 'Linux' }} + env: + PY: '3.11' + PROJECT_NAME: taichi + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_BACKTRACE:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + TI_WANTED_ARCHS: 'cpu,cuda,vulkan,opengl,gles' + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '1' + + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + env: + EXTRA_TEST_MARKERS: ${{ matrix.extra_markers }} + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + build_and_test_manylinux2014: + name: Build and Test (manylinux2014) + needs: check_files + runs-on: [self-hosted, cuda, cn, release] + env: + PY: "3.8" + PROJECT_NAME: taichi + TI_WANTED_ARCHS: "cuda,cpu" + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=OFF + -DTI_BUILD_TESTS:BOOL=ON + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-build-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-manylinux2014-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + + build_and_test_amdgpu_linux: + name: Build and Test (AMDGPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + runs-on: [self-hosted, amdgpu] + env: + PY: '3.8' + PROJECT_NAME: taichi + TI_WANTED_ARCHS: 'cpu,amdgpu' + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '0' + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_CUDA:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=OFF + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_AMDGPU:BOOL=ON + + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-amdgpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-amdgpu:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-amdgpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-amdgpu:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_test.sh + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + + build_and_test_windows: + name: Build and Test Windows + needs: check_files + strategy: + matrix: + extra_markers: + - sm70 + - not sm70 + runs-on: + - self-hosted + - cn + - windows + - cuda + - OpenGL + - ${{ matrix.extra_markers == 'sm70' && 'sm70' || 'windows' }} + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 180 }} + env: + PY: "3.7" + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_DX11:BOOL=ON + -DTI_WITH_DX12:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_BACKTRACE=ON + -DTI_WITH_C_API=ON + TI_WANTED_ARCHS: cpu,cuda,vulkan,opengl,gles + TI_SKIP_VERSION_CHECK: ON + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '1' + + steps: + - name: Workaround checkout Needed single revision issue + run: | + git config --system core.longpaths true + git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' # no '|| true' here + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: + python-version: 3.7 + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + shell: cmd + run: | + python .\.github\workflows\scripts\build.py + + - name: Test + id: test + shell: pwsh + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + .\.github\workflows\scripts\win_test.ps1 -libsDir "$env:LocalAppData/buildbot" + env: + EXTRA_TEST_MARKERS: ${{ matrix.extra_markers }} + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Cleanup Git Cache Configs + shell: pwsh + if: always() + run: | + . .github/workflows/scripts/common-utils.ps1 + UnsetGitCachingProxy + exit 0 + + build_and_test_m1: + name: Build and Test (Apple M1) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 60 || 120 }} + strategy: + matrix: + include: + - os: macos-latest + python: 3.8 + defaults: + run: + # https://github.com/actions/runner/issues/805#issuecomment-844426478 + shell: '/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}' + runs-on: [self-hosted, m1] + env: + CXX: clang++ + PY: ${{ matrix.python }} + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=OFF + -DTI_WITH_CUDA:BOOL=OFF + -DTI_WITH_VULKAN:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_C_API=ON + -DTI_WITH_STATIC_C_API=ON + TI_WANTED_ARCHS: 'cpu,metal,vulkan' + PLATFORM: 'm1' + TI_RUN_RELEASE_TESTS: '1' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + brew install molten-vk + brew install llvm@15 + .github/workflows/scripts/build.py + env: + CXX: clang++ + + - name: Check C-API Export Symbols + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + + .github/workflows/scripts/aot-demo.sh check-c-api-export-symbols + + - name: Test + id: test + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + .github/workflows/scripts/unix_test.sh + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 + + + build_ios_capi: + name: Build iOS C-API Static Library + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 60 || 120 }} + runs-on: [self-hosted, m1] + env: + PY: "3.10" + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + fetch-depth: '0' + submodules: 'recursive' + + - name: Build + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: .github/workflows/scripts/build.py ios + + - name: Save Compiled Static Library + if: ${{ needs.check_files.outputs.run_job != 'false' }} + uses: actions/upload-artifact@v3 + with: + name: libtaichi_c_api.iOS.a + path: 'dist/C-API-iOS/*.a' + retention-days: 7 + + build_and_test_docs: + name: Build and Test (Docs) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + runs-on: [self-hosted, Linux, cuda, vulkan, cn] + env: + PY: '3.8' + PROJECT_NAME: taichi + TI_WANTED_ARCHS: 'cpu,cuda,vulkan,opengl,gles' + TI_DEVICE_MEMORY_GB: '1' + TI_RUN_RELEASE_TESTS: '1' + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_BACKTRACE:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_C_API=ON + + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Check if there is anything Markdown modified but not covered by CI + if: ${{ github.event_name == 'pull_request' }} + run: | + set +e + sort docs/cover-in-ci.lst > /tmp/sorted-cover-list + git diff --name-only $BASE_SHA $HEAD_SHA | grep '\.md$' | sort > /tmp/sorted-modified-list + diff -u /tmp/sorted-cover-list /tmp/sorted-modified-list | grep '^\+docs' > /tmp/uncovered-list + if [ -s /tmp/uncovered-list ]; then + echo Please add modified docs to docs/cover-in-ci.lst! + echo + cat /tmp/uncovered-list + exit 1 + else + echo No modified docs, or modified docs are all covered in the list + fi + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Test + id: test + run: | + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-test-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix-test-docs.sh + + aot_build_and_test_android_apps: + name: AOT Build and Test Android Apps + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API:BOOL=ON + + - name: Test For Android AOT + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run-gpu --name taichi-test-android \ + registry.botmaster.tgr/taichi-test-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-smoke-test-android-aot-demo + + env: + TI_DEVICE_MEMORY_GB: '0.1' + + aot_build_and_test_android_headless_demos: + name: AOT Build and Test Android Headless Demos + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API=ON + + - name: Build & Run C-API Headless Demos (Android) + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run-gpu --name taichi-test-capi-headless-demo \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-test-headless-demo + env: + TI_DEVICE_MEMORY_GB: '0.1' + + aot_build_and_test_android_unity_examples: + name: AOT Build and Test Android Unity Examples + # Skip this job when testing the offline cache + if: ${{ github.event.schedule != '0 18 * * *' }} + needs: check_files + runs-on: [self-hosted, Linux, cn] + timeout-minutes: 60 + permissions: + packages: read + contents: read + env: + REDIS_HOST: 172.16.5.1 + PY: '3.9' + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + name: Checkout taichi + with: + fetch-depth: '0' + submodules: "recursive" + + - name: Prepare Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: >- + . .github/workflows/scripts/common-utils.sh; + prepare-build-cache; + + TAICHI_WHEEL_DIR=$(mktemp -d); + echo TAICHI_WHEEL_DIR=$TAICHI_WHEEL_DIR >> $GITHUB_ENV; + chmod 0777 $TAICHI_WHEEL_DIR; + + echo CI_DOCKER_RUN_EXTRA_ARGS=" + -e REDIS_HOST + -v $(pwd):/home/dev/taichi + -v $TAICHI_WHEEL_DIR:/taichi-wheel + " >> $GITHUB_ENV; + + - name: Build Host Taichi + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-host \ + -v $TAICHI_WHEEL_DIR:/home/dev/taichi/dist \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_C_API=OFF + + - name: Build For Android + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + git clean -fxd + ci-docker-run --name taichi-build-android \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py android + env: + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_LLVM:BOOL=OFF + -DTI_WITH_C_API=ON + + - name: Prepare Unity Build Environment + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-prepare-unity-build-env \ + registry.botmaster.tgr/taichi-build-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh prepare-unity-build-env + + - name: Build Taichi-UnityExample + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-build-unity-demo \ + registry.taichigraphics.com/unityci-editor:ubuntu-2020.3.14f1-android-1-with-secret-sauce \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-unity-demo + + - name: Run Taichi-UnityExample (C-API) + if: ${{ needs.check_files.outputs.run_job != 'false' }} + run: | + . .github/workflows/scripts/common-utils.sh + ci-docker-run --name taichi-run-unity-demo \ + registry.botmaster.tgr/taichi-test-android:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh smoke-test-unity-demo + + aot_build_and_test_gpu_linux: + name: AOT Build and Test (GPU) + needs: check_files + timeout-minutes: ${{ github.event.schedule != '0 18 * * *' && 90 || 120 }} + # TODO(proton): do we need to test both drivers? + strategy: + matrix: + tags: + - [self-hosted, cuda, vulkan, cn, driver470] + - [self-hosted, cuda, vulkan, cn, driver510] + env: + PY: '3.8' + PROJECT_NAME: taichi + TAICHI_CMAKE_ARGS: >- + -DTI_WITH_OPENGL:BOOL=ON + -DTI_WITH_VULKAN:BOOL=ON + -DTI_WITH_BACKTRACE:BOOL=ON + -DTI_BUILD_TESTS:BOOL=ON + -DTI_WITH_C_API=ON + + runs-on: ${{ matrix.tags }} + steps: + - name: Workaround checkout Needed single revision issue + run: git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD' || true + + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + fetch-depth: '0' + + - name: Prepare Environment + run: | + . .github/workflows/scripts/common-utils.sh + prepare-build-cache + echo CI_DOCKER_RUN_EXTRA_ARGS="-v $(pwd):/home/dev/taichi" >> $GITHUB_ENV + + - name: Build & Install + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-build \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/build.py + + - name: Check C-API Export Symbols + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test-check-c-api-export-symbols \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh check-c-api-export-symbols + + - name: Build & Run C-API Headless Demos (Desktop) + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + chown -R 1000:1000 . + ci-docker-run-gpu --name taichi-test-capi-headless-demo-desktop \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/aot-demo.sh build-and-test-headless-demo-desktop + env: + TI_DEVICE_MEMORY_GB: '0.1' + + - name: Compatibility Test + id: test + run: | + [[ ${{needs.check_files.outputs.run_job}} == false ]] && exit 0 + . .github/workflows/scripts/common-utils.sh + + ci-docker-run-gpu --name taichi-test \ + registry.botmaster.tgr/taichi-build-cuda:${{ env.CI_IMAGE_VERSION }} \ + /home/dev/taichi/.github/workflows/scripts/unix_aot_compat_test.sh + + - name: Save wheel if test failed + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: broken-wheel + path: dist/* + retention-days: 7 + + - name: Save Bad Captures + if: failure() && steps.test.conclusion == 'failure' + uses: actions/upload-artifact@v3 + with: + name: bad-captures + path: taichi-release-tests/bad-compare/* + retention-days: 7 diff --git a/.github/workflows/scripts/ti_build/cmake.py b/.github/workflows/scripts/ti_build/cmake.py index cf6d07ed5..c07c026cb 100644 --- a/.github/workflows/scripts/ti_build/cmake.py +++ b/.github/workflows/scripts/ti_build/cmake.py @@ -14,7 +14,7 @@ from .misc import banner # -- code -- -OPTION_RE = re.compile(r'option\(([A-Z0-9_]*) +"(.*?)" +(ON|OFF)\)') +OPTION_RE = re.compile(r'option\(([A-Z0-9_]*) +"(.*?)" +(ON|OFF)\)(?: *# wheel-tag: (.*))?') DEF_RE = re.compile(r"-D([A-Z0-9_]*)(?::BOOL)?=([^ ]+)(?: |$)") @@ -28,10 +28,7 @@ def __init__(self, environ_name): self.environ_name = environ_name self.definitions = {} self.option_definitions = { - "CMAKE_EXPORT_COMPILE_COMMANDS": ( - "Generate compile_commands.json", - False, - ), + "CMAKE_EXPORT_COMPILE_COMMANDS": ("Generate compile_commands.json", False, ""), } self.finalized = False @@ -39,9 +36,9 @@ def __init__(self, environ_name): def collect_options(self, *files: str) -> None: for fn in files: with open(fn, "r") as f: - for name, desc, default in OPTION_RE.findall(f.read()): + for name, desc, default, wheel_tag in OPTION_RE.findall(f.read()): default = self._VMAP.get(default, default) - self.option_definitions[name] = (desc, default) + self.option_definitions[name] = (desc, default, wheel_tag) def parse_initial_args(self) -> None: args = os.environ.get(self.environ_name, "") @@ -49,7 +46,7 @@ def parse_initial_args(self) -> None: self.set(name, value) def get_effective(self, name: str) -> Union[str, bool]: - _, default = self.option_definitions.get(name, ("", None)) + _, default, _ = self.option_definitions.get(name, ("", None, "")) return self.definitions.get(name, default) def set(self, name: str, value: Union[str, bool]) -> None: @@ -57,7 +54,7 @@ def set(self, name: str, value: Union[str, bool]) -> None: desc = "" value = self._VMAP.get(value, value) default = None - desc, default = self.option_definitions.get(name, ("", None)) + desc, default, wheel_tag = self.option_definitions.get(name, ("", None, "")) desc = desc and f" ({desc}) " is_bool = isinstance(default, bool) assert not is_bool or isinstance(value, bool), f"Option {name} must be bool" @@ -84,6 +81,7 @@ def set(self, name: str, value: Union[str, bool]) -> None: else: p(f"{B}:: CMAKE: Already disabled: {name}{desc}{N}") else: + assert not wheel_tag, "Set a non boolean value to an option with wheel-tag" if orig != value: if orig != default: p(f"{R}:: CMAKE- {name}={orig}{desc}{N}") @@ -99,7 +97,7 @@ def render(self) -> List[Tuple[str, str, str]]: else: v = f"-D{name}={value}" - desc, _ = self.option_definitions.get(name, ("", None)) + desc, _, _ = self.option_definitions.get(name, ("", None, "")) if desc: prefix = "DO NOT " if not value else "" desc = f" ({prefix}{desc})" @@ -108,6 +106,16 @@ def render(self) -> List[Tuple[str, str, str]]: return lst + def render_wheel_tag(self) -> str: + tags = [] + for name, (_, default, wheel_tag) in self.option_definitions.items(): + if not wheel_tag: + continue + print(name, self.definitions) + if self.definitions.get(name, default): + tags.append(wheel_tag) + return ".".join(sorted(tags)) + @banner("{self.environ_name} Summary") def print_summary(self, rendered) -> None: p = lambda s: print(s, file=sys.stderr, flush=True) diff --git a/.github/workflows/scripts/ti_build/entry.py b/.github/workflows/scripts/ti_build/entry.py index da62d0265..654583176 100644 --- a/.github/workflows/scripts/ti_build/entry.py +++ b/.github/workflows/scripts/ti_build/entry.py @@ -2,6 +2,7 @@ # -- stdlib -- import argparse +import datetime import os import platform import subprocess @@ -31,14 +32,18 @@ def build_wheel(python: Command, pip: Command) -> None: Build the Taichi wheel """ git.fetch("origin", "master", "--tags") - proj = os.environ.get("PROJECT_NAME", "taichi") proj_tags = [] extra = [] - if proj == "taichi-nightly": - proj_tags.extend(["egg_info", "--tag-date", "--tag-build=.post"]) - # Include C-API in nightly builds - cmake_args["TI_WITH_C_API"] = True + cmake_args.writeback() + wheel_tag = cmake_args.render_wheel_tag() + + if misc.options.nightly: + os.environ["PROJECT_NAME"] = "taichi-nightly" + now = datetime.datetime.now().strftime("%Y%m%d") + proj_tags.extend(["egg_info", f"--tag-build=.post{now}+{wheel_tag}"]) + elif misc.options.tag_config: + proj_tags.extend(["egg_info", f"--tag-build=+{wheel_tag}"]) if platform.system() == "Linux": if is_manylinux2014(): @@ -46,7 +51,6 @@ def build_wheel(python: Command, pip: Command) -> None: else: extra.extend(["-p", "manylinux_2_27_x86_64"]) - cmake_args.writeback() python("setup.py", "clean") python("misc/make_changelog.py", "--ver", "origin/master", "--repo_dir", "./", "--save") @@ -151,6 +155,12 @@ def parse_args(): help = "Continue when encounters error." parser.add_argument("--permissive", action="store_true", default=False, help=help) + help = "Tag built wheel with TI_WITH_xxx config." + parser.add_argument("--tag-config", action="store_true", default=False, help=help) + + help = "Build nightly wheel." + parser.add_argument("--nightly", action="store_true", default=False, help=help) + options = parser.parse_args() return options diff --git a/CMakeLists.txt b/CMakeLists.txt index 127852b51..9bc06b96e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) option(USE_LLD "Use lld (from llvm) linker" OFF) option(USE_MOLD "Use mold (A Modern Linker)" OFF) -option(TI_WITH_BACKTRACE "Use backward-cpp to print out C++ stack trace upon failure" OFF) +option(TI_WITH_BACKTRACE "Use backward-cpp to print out C++ stack trace upon failure" OFF) # wheel-tag: bt if(LINUX OR APPLE) if (NOT IOS) @@ -205,8 +205,8 @@ endif() configure_file(taichi/common/version.h.in ${CMAKE_SOURCE_DIR}/taichi/common/version.h) configure_file(taichi/common/commit_hash.h.in ${CMAKE_SOURCE_DIR}/taichi/common/commit_hash.h) -option(TI_WITH_C_API "build taichi runtime c-api library" ON) -option(TI_WITH_STATIC_C_API "build static taichi runtime c-api library" OFF) +option(TI_WITH_C_API "build taichi runtime c-api library" ON) # wheel-tag: aot +option(TI_WITH_STATIC_C_API "build static taichi runtime c-api library" OFF) # wheel-tag: static_aot if(TI_WITH_STATIC_C_API) set(TI_WITH_C_API ${TI_WITH_STATIC_C_API}) @@ -231,7 +231,7 @@ if (TI_BUILD_RHI_EXAMPLES) endif() -option(TI_WITH_GRAPHVIZ "generate dependency graphs between targets" OFF) +option(TI_WITH_GRAPHVIZ "generate dependency graphs between targets" OFF) # wheel-tag: viz if (TI_WITH_GRAPHVIZ) set(GRAPHVIZ_GRAPH_NAME "ti_targets") add_custom_target(graphviz ALL diff --git a/cmake/TaichiCore.cmake b/cmake/TaichiCore.cmake index 8579d2d3b..a3b44521f 100644 --- a/cmake/TaichiCore.cmake +++ b/cmake/TaichiCore.cmake @@ -1,14 +1,14 @@ option(USE_STDCPP "Use -stdlib=libc++" OFF) -option(TI_WITH_LLVM "Build with LLVM backends" ON) -option(TI_WITH_METAL "Build with the Metal backend" ON) -option(TI_WITH_CUDA "Build with the CUDA backend" ON) -option(TI_WITH_CUDA_TOOLKIT "Build with the CUDA toolkit" OFF) -option(TI_WITH_AMDGPU "Build with the AMDGPU backend" OFF) -option(TI_WITH_OPENGL "Build with the OpenGL backend" ON) -option(TI_WITH_VULKAN "Build with the Vulkan backend" OFF) -option(TI_WITH_DX11 "Build with the DX11 backend" OFF) -option(TI_WITH_DX12 "Build with the DX12 backend" OFF) -option(TI_WITH_GGUI "Build with GGUI" OFF) +option(TI_WITH_LLVM "Build with LLVM backends" ON) # wheel-tag: llvm +option(TI_WITH_METAL "Build with the Metal backend" ON) # wheel-tag: mtl +option(TI_WITH_CUDA "Build with the CUDA backend" ON) # wheel-tag: cu +option(TI_WITH_CUDA_TOOLKIT "Build with the CUDA toolkit" OFF) # wheel-tag: cutk +option(TI_WITH_AMDGPU "Build with the AMDGPU backend" OFF) # wheel-tag: amd +option(TI_WITH_OPENGL "Build with the OpenGL backend" ON) # wheel-tag: gl +option(TI_WITH_VULKAN "Build with the Vulkan backend" OFF) # wheel-tag: vk +option(TI_WITH_DX11 "Build with the DX11 backend" OFF) # wheel-tag: dx11 +option(TI_WITH_DX12 "Build with the DX12 backend" OFF) # wheel-tag: dx12 +option(TI_WITH_GGUI "Build with GGUI" OFF) # wheel-tag: ggui # Force symbols to be 'hidden' by default so nothing is exported from the Taichi # library including the third-party dependencies.