[CI] Linux CD containerization (#107)

.github/workflows/release.yml

@@ -20,9 +20,9 @@ jobs:
       - id: set-matrix
         run: |
           # For nightly release, we only run on python 3.8
-          [ -z "${{ github.event.action }}" ] && matrix="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\"}]"
+          [ -z "${{ github.event.action }}" ] && matrix="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\",\"conda_python\":\"py38\"}]"
           # For production release, we run on four python versions.
-          [ -z "${{ github.event.action }}" ] || matrix="[{\"name\":\"taichi\",\"python\":\"3.6\"},{\"name\":\"taichi\",\"python\":\"3.7\"},{\"name\":\"taichi\",\"python\":\"3.8\"},{\"name\":\"taichi\",\"python\":\"3.9\"}]"
+          [ -z "${{ github.event.action }}" ] || matrix="[{\"name\":\"taichi\",\"python\":\"3.6\",\"conda_python\":\"py36\"},{\"name\":\"taichi\",\"python\":\"3.7\",\"conda_python\":\"py37\"},{\"name\":\"taichi\",\"python\":\"3.8\",\"conda_python\":\"py38\"},{\"name\":\"taichi\",\"python\":\"3.9\",\"conda_python\":\"py39\"}]"
           echo ::set-output name=matrix::{\"include\":$(echo $matrix)}\"
           # M1 only supports py38 and py39(conda), so change matrix.
           [ -z "${{ github.event.action }}" ] && matrix_osx="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\"}]"
@@ -41,56 +41,36 @@ jobs:
         with:
           submodules: "recursive"
 
-      - name: Create Python Wheel
+      - name: Build
         run: |
-          # We hacked here because conda activate in CI won't update python PATH
-          # automatically. So we don't activate and use desired python version
-          # directly.
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
-          TAICHI_REPO_DIR=`pwd`
-          export PATH=$LLVM_LIB_ROOT_DIR/bin:/usr/local/cuda/bin:$PATH
-          export LLVM_DIR=$LLVM_LIB_ROOT_DIR/lib/cmake/llvm
-          export CXX=clang++-8
-          python3 -m pip uninstall taichi taichi-nightly -y
-          python3 -m pip install -r requirements_dev.txt
-          python3 -m pip install twine
-          cd python
-          git fetch origin master
-          export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
-          python3 build.py build --project_name $PROJECT_NAME
-          cd ..
-          NUM_WHL=`ls dist/*.whl | wc -l`
-          if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
-          python3 -m pip install dist/*.whl
+          mkdir -m777 wheel
+          docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
+          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/
+          docker start -a taichi_build
         env:
-          LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
-          BUILD_NUM_THREADS: 8
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
+          PY: ${{ matrix.conda_python }}
+          GPU_BUILD: ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
-          PYTHON: ${{ matrix.python }}
+          DISPLAY: :1
 
       - name: Archive Wheel Artifacts
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.name }}-py${{ matrix.python }}-linux.whl
-          path: dist/*.whl
+          path: wheel/*.whl
           retention-days: 20
 
       - name: Test
         run: |
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
-          python3 examples/algorithm/laplace.py
-          export DISPLAY=:1
-          hash -r
-          glewinfo
-          ti diagnose
-          ti changelog
-          ti test -vr2 -t2 -k "not ndarray and not torch"
-          # ndarray test might OOM if run with -t2.
-          # FIXME: unify this with presubmit.yml to avoid further divergence
-          ti test -vr2 -t1 -k "ndarray or torch"
+          docker create --user dev --name taichi_test --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/unix_docker_test.sh $PY $GPU_TEST
+          docker cp .github/workflows/scripts/unix_docker_test.sh taichi_test:/home/dev/unix_docker_test.sh
+          docker cp wheel/*.whl taichi_test:/home/dev/
+          docker start -a taichi_test
         env:
-          PYTHON: ${{ matrix.python }}
+          PY: py38
+          GPU_TEST: ON
+          DISPLAY: :1
 
       - name: Upload PyPI
         env:
@@ -100,11 +80,17 @@ jobs:
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
         run: |
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
+          mkdir dist
+          cp wheel/*.whl dist/
           cd python
           if [ $PROJECT_NAME == "taichi-nightly" ]; then export PYPI_PWD="$NIGHT_PWD" && python3 build.py upload --skip_build --testpypi --project_name $PROJECT_NAME
           elif [ $PROJECT_NAME == "taichi" ]; then export PYPI_PWD="$PROD_PWD" && python3 build.py upload --skip_build; fi
 
+      - name: clean docker container
+        if: always()
+        run: |
+          docker rm taichi_build taichi_test -f
+
   build_and_upload_mac:
     name: Build and Upload (macOS only)
     needs: matrix_prep
@@ -132,6 +118,7 @@ jobs:
           export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
           export CXX=clang++
           python -m pip install -r requirements_dev.txt
+          python -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -141,7 +128,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
           PROJECT_NAME: ${{ matrix.name }}
 
       - name: Archive Wheel Artifacts
@@ -155,7 +142,7 @@ jobs:
         run: |
           python examples/algorithm/laplace.py
           ti diagnose
-          ti test -vr2 -t2
+          ti test -vr2 -t2 -a cpu,metal
 
       - name: Upload PyPI
         env:
@@ -193,6 +180,7 @@ jobs:
           git --version
           export CXX=clang++
           python3 -m pip install -r requirements_dev.txt
+          python3 -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -202,7 +190,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           python3 -m pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
 
@@ -250,6 +238,7 @@ jobs:
           git --version
           export CXX=clang++
           python3 -m pip install -r requirements_dev.txt
+          python3 -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -259,7 +248,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           python3 -m pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
 
@@ -336,6 +325,7 @@ jobs:
           clang --version
           cd D:\a\taichi\taichi
           python -m pip install -r requirements_dev.txt
+          python -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           $env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS

.github/workflows/scripts/unix_docker_build.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+PY=$1
+GPU_BUILD=$2
+PROJECT_NAME=$3
+CI_SETUP_CMAKE_ARGS=$4
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+python3 -m pip uninstall taichi taichi-nightly -y
+
+cd taichi
+
+if [[ $GPU_BUILD == "OFF" ]]
+then 
+    python3 -m pip install -r requirements_dev.txt
+fi
+
+cd python
+# This is for changelog
+git fetch origin master
+TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build --project_name $PEOJECT_NAME
+# Run basic cpp tests
+cd ..
+CUR_DIR=`pwd`
+TI_LIB_DIR=$CUR_DIR/python/taichi/lib ./build/taichi_cpp_tests
+
+cp dist/*.whl /wheel/

.github/workflows/scripts/unix_docker_test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+PY=$1
+GPU_TEST=$2
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+python3 -m pip install ./*.whl
+
+if [[ $GPU_TEST == "OFF" ]]
+then
+    python3 -m pip install -r requirements_test.txt
+fi
+
+export TI_IN_DOCKER=true
+ti diagnose
+ti changelog
+
+if [[ $GPU_TEST == "OFF" ]]
+then
+    ti test -vr2 -t2
+else
+    ti test -vr2 -t2 -k "not ndarray and not torch"
+    ti test -vr2 -t1 -k "ndarray or torch"
+fi

.github/workflows/scripts/unix_test.sh

@@ -1,5 +1,6 @@
 set -ex
 TAICHI_REPO_DIR=`pwd`
+python3 -m pip install -r requirements_test.txt
 TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
 [[ $RUN_CPP_TESTS == "ON" ]] && TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
 export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH