From 02700850f733f126e794a614e5d68cb1335e879d Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Tue, 19 Oct 2021 14:35:39 +0800
Subject: [PATCH 01/24] use docker CI (#99)

---
 .github/workflows/presubmit.yml | 294 +-------------------------------
 1 file changed, 9 insertions(+), 285 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 419820005..1c3b74f69 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -4,292 +4,16 @@ on:
     types: [opened, synchronize, reopened]
 
 jobs:
-  title_format:
-    name: Check PR Title
-    if: ${{ github.event.pull_request }}
+  docker_cpu_build:
     runs-on: ubuntu-latest
+    container: registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
+      - run: |
+        ti test
 
-      - name: Run PR Title Checker
-        run: |
-          pip install semver GitPython
-          python misc/ci_check_pr_title.py "$PR_TITLE"
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
-
-  check_code_format:
-    name: Check Code Format
-    runs-on: ubuntu-latest
-    # This job will be required to pass before merging to master branch.
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Setup git & clang-format
-        run: |
-          git config user.email "taichigardener@gmail.com"
-          git config user.name "Taichi Gardener"
-          git checkout -b _fake_squash
-          git remote add upstream https://github.com/taichi-dev/taichi.git
-          git fetch upstream master
-          sudo apt install clang-format-10
-
-      - name: Cache PIP
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ hashFiles('setup.py') }}-${{ hashFiles('requirements_dev.txt') }}
-
-      - name: Install requirements
-        run: |
-          python3 -m pip install --user -r requirements_dev.txt
-
-      - name: Check code format
-        run: |
-          python3 misc/code_format.py
-          git checkout -b _enforced_format
-          git commit -am "enforce code format" || true
-          # exit with 1 if there were differences:
-          git diff _fake_squash _enforced_format --exit-code
-
-      - name: Pylint
-        run: |
-          # Make sure pylint doesn't regress
-          pylint python/taichi/ --disable=all --enable=C0415
-          if [ $? -eq 0 ]
-          then
-            echo "PASSED: pylint is happy"
-            exit 0
-          else
-            echo "FAILED: please run the pylint command above and make sure it passes"
-            exit 1
-          fi
-
-  build_and_test_cpu_required:
-    # This job will be required to pass before merging to master branch.
-    name: Required Build and Test (CPU)
-    needs: check_code_format
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            python: 3.6
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
-
-      - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-
-  build_and_test_cpu:
-    name: Build and Test (CPU)
-    needs: build_and_test_cpu_required
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.7
-            with_cc: OFF
-            with_cpp_tests: ON
-          - os: ubuntu-latest
-            python: 3.9
-            with_cc: OFF
-            with_cpp_tests: OFF
-          - os: ubuntu-latest
-            python: 3.8
-            with_cc: ON
-            with_cpp_tests: OFF
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
-
-      - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
-          CXX: clang++
-      # [DEBUG] Copy this step around to enable debugging inside Github Action instances.
-      #- name: Setup tmate session
-      #  uses: mxschmitt/action-tmate@v3
-      #  with:
-      #    limit-access-to-actor: true
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          RUN_CPP_TESTS: ${{ matrix.with_cpp_tests }}
-
-  build_and_test_gpu_linux:
-    name: Build and Test (GPU)
-    needs: check_code_format
-    runs-on: [self-hosted, cuda, vulkan, cn]
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - name: Build
-        run: |
-          export PATH=$PATH:/usr/local/cuda/bin
-          .github/workflows/scripts/unix_build.sh
-        env:
-          LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON
-          BUILD_NUM_THREADS: 8
-          LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
-          LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
-          CXX: clang++-8
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          DISPLAY: :1
-          GPU_TEST: ON
-
-  build_and_test_windows:
-    name: Build and Test (Windows)
-    needs: check_code_format
-    runs-on: windows-latest
-    timeout-minutes: 90
-    steps:
-
-      - name: Install 7Zip PowerShell
-        shell: powershell
-        run: Install-Module 7Zip4PowerShell -Force -Verbose
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Add msbuild to PATH
-        uses: microsoft/setup-msbuild@v1.0.2
-
-      - name: Download And Install Vulkan
-        shell: powershell
-        run: |
-          Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
-          $installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
-          $installer.WaitForExit();
-
-      - name: Build
-        shell: powershell
-        run: |
-          $env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
-          cd C:\
-          Remove-item alias:curl
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
-          7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
-          7z x clang-10.0.0-win.zip -otaichi_clang
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          clang --version
-          cd D:\a\taichi\taichi
-          python -m pip install -r requirements_dev.txt
-          cd python
-          git fetch origin master
-          $env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
-          python build.py build
-          cd ..\dist
-          $env:WHL = $(dir *.whl)
-          python -m pip install $env:WHL
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-          CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
-          VULKAN_SDK: C:/VulkanSDK/1.2.189.0
-
-      - name: Test
-        shell: powershell
-        run: |
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          python -c "import taichi"
-          python examples/algorithm/laplace.py
-          python bin/taichi diagnose
-          python bin/taichi changelog
-          python bin/taichi test -vr2 -t2
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-
-  build_and_test_m1:
-    name: Build and Test (Apple M1)
-    needs: check_code_format
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.8
-    defaults:
-      run:
-        # https://github.com/actions/runner/issues/805#issuecomment-844426478
-        shell: "/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}"
-    runs-on: [self-hosted, m1]
+  docker_gpu_build:
+    runs-on: [self-hosted, cuda, vulkan]
+    container: registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
     steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - name: Build
-        run: |
-          rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
-          .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
-
-      - name: Test
-        run: |
-          export PATH=$PATH:$HOME/Library/Python/3.8/bin
-          python3 examples/algorithm/laplace.py
-          TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
-          TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
-          ti test -vr2 -t4 -x
+      - run: |
+        ti test

From a53792a710aafb8461df74a7b2dcb809a80ba224 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Thu, 21 Oct 2021 17:43:01 +0800
Subject: [PATCH 02/24] use docker CI (#100)

---
 .github/workflows/presubmit.yml               | 34 +++++++++++++------
 .../workflows/scripts/unix_docker_build.sh    | 29 ++++++++++++++++
 2 files changed, 53 insertions(+), 10 deletions(-)
 create mode 100755 .github/workflows/scripts/unix_docker_build.sh

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 1c3b74f69..60c95486a 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -4,16 +4,30 @@ on:
     types: [opened, synchronize, reopened]
 
 jobs:
-  docker_cpu_build:
-    runs-on: ubuntu-latest
-    container: registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
-    steps:
-      - run: |
-        ti test
 
-  docker_gpu_build:
-    runs-on: [self-hosted, cuda, vulkan]
-    container: registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
+  docker_cpu_test:
+    runs-on: [self-hosted, cn, cuda, vulkan]
     steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+
       - run: |
-        ti test
+          docker create --user dev --name taichi --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu20.04:v0.1.0 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh CI_SETUP_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS PY=$PY
+          docker cp ./taichi taichi:/home/dev/taichi
+          docker start -a taichi
+        env:
+          DISPLAY: :1
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
+          PY: py38
+
+      - name: clean docker container
+        if: always()
+        run: docker rm taichi -f
+
+      ##- uses: addnab/docker-run-action@v3
+      ##  with:
+      ##    username: admin
+
+          #- uses: docker://registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
+
diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh
new file mode 100755
index 000000000..d4b284bba
--- /dev/null
+++ b/.github/workflows/scripts/unix_docker_build.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+for ARGUMENT in "$@"
+do
+    KEY=$(echo $ARGUMENT | cut -f1 -d=)
+    VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+    case "$KEY" in
+            CI_SETUP_CMAKE_ARGS)              SHA=${VALUE} ;;
+            PY)               PY=${VALUE} ;;
+            *)
+    esac
+done
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+cd taichi
+python3 -m pip install --user -r requirements_dev.txt
+python3 -m pip install torch==1.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 setup.py install --user
+
+export TI_IN_DOCKER=true
+
+ti diagnose
+ti test -vr2 -t2 -k "not ndarray and not torch"
+ti test -vr2 -t1 -k "ndarray or torch"

From 092e094d8b2aeeb455dca96bf76763657081129a Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 25 Oct 2021 23:01:37 +0800
Subject: [PATCH 03/24] [ci] Test auto benchmark

---
 .github/workflows/presubmit.yml   | 101 +++++++++++++++---
 benchmarks/misc/membound.py       | 165 ++++++++++++++++++------------
 benchmarks/misc/membound_cases.py |   8 +-
 benchmarks/misc/run.py            | 125 ++++++++++++++++++----
 benchmarks/misc/utils.py          |  43 ++++++--
 5 files changed, 323 insertions(+), 119 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 60c95486a..252a9123a 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -1,3 +1,5 @@
+
+  
 name: Presubmit Checks
 on:
   pull_request:
@@ -5,29 +7,94 @@ on:
 
 jobs:
 
-  docker_cpu_test:
-    runs-on: [self-hosted, cn, cuda, vulkan]
+  title_format:
+    name: Check PR Title
+    if: ${{ github.event.pull_request }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
         with:
-          submodules: 'recursive'
+          python-version: 3.8
 
-      - run: |
-          docker create --user dev --name taichi --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu20.04:v0.1.0 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh CI_SETUP_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS PY=$PY
-          docker cp ./taichi taichi:/home/dev/taichi
-          docker start -a taichi
+      - name: Run PR Title Checker
+        run: |
+          pip install semver GitPython
+          python misc/ci_check_pr_title.py "$PR_TITLE"
         env:
-          DISPLAY: :1
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
-          PY: py38
+          PR_TITLE: ${{ github.event.pull_request.title }}
+
+  check_code_format:
+    name: Check Code Format
+    runs-on: ubuntu-latest
+    # This job will be required to pass before merging to master branch.
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
 
-      - name: clean docker container
-        if: always()
-        run: docker rm taichi -f
+      - name: Setup git & clang-format
+        run: |
+          git config user.email "taichigardener@gmail.com"
+          git config user.name "Taichi Gardener"
+          git checkout -b _fake_squash
+          git remote add upstream https://github.com/taichi-dev/taichi.git
+          git fetch upstream master
+          sudo apt install clang-format-10
+      - name: Cache PIP
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ hashFiles('setup.py') }}-${{ hashFiles('requirements_dev.txt') }}
 
-      ##- uses: addnab/docker-run-action@v3
-      ##  with:
-      ##    username: admin
+      - name: Install requirements
+        run: |
+          python3 -m pip install --user -r requirements_dev.txt
+      - name: Check code format
+        run: |
+          python3 misc/code_format.py
+          git checkout -b _enforced_format
+          git commit -am "enforce code format" || true
+          # exit with 1 if there were differences:
+          git diff _fake_squash _enforced_format --exit-code
+  performance_monitoring:
+    name: Performance monitoring (NVGPU)
+    needs: check_code_format
+    timeout-minutes: 60
+    runs-on: [self-hosted, x64, cuda, linux, benchmark]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
 
-          #- uses: docker://registry.taichigraphics.com/taichidev-ubuntu20.04-test:v0.1
+      - name: Build & Install
+        run: |
+          export PATH=$PATH:/usr/local/cuda/bin
+          .github/workflows/scripts/unix_build.sh
+        env:
+          LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
+          LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
+          LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
+          CUDA_TOOLKIT_ROOT_DIR: /usr/local/cuda/
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON
+          BUILD_NUM_THREADS: 8
+          CXX: clang++-10
 
+      - name: Run benchmark
+        run: |
+          # in Taichi dir
+          export TAICHI_REPO_DIR=`pwd`
+          export BENCHMARK_CODE_DIR=$TAICHI_REPO_DIR/benchmarks/misc/
+          echo $TAICHI_REPO_DIR
+          echo $BENCHMARK_CODE_DIR
+          cd ..
+          # update benchmark code
+          git cloen git@github.com:yolo2themoon/perf_monitoring.git
+          cd perf_monitoring
+          export PERF_MONITORING_REPO_DIR=`pwd`
+          echo $PERF_MONITORING_REPO_DIR
+          cp -r $BENCHMARK_CODE_DIR/* $PERF_MONITORING_REPO_DIR/benchmarks/
+          # run benchmark
+          ./run.sh
\ No newline at end of file
diff --git a/benchmarks/misc/membound.py b/benchmarks/misc/membound.py
index 55bfd3c99..fe719dc9a 100644
--- a/benchmarks/misc/membound.py
+++ b/benchmarks/misc/membound.py
@@ -1,107 +1,138 @@
+import json
+import os
 import time
 
 from membound_cases import fill, reduction, saxpy
-from utils import *
+from utils import (arch_name, dtype2str, dump2json, geometric_mean, kibibyte,
+                   md_table_header, repeat_times, size2str)
 
 import taichi as ti
 
-test_cases = [fill, saxpy, reduction]
-test_archs = [ti.cuda]
-test_dtype = [ti.i32, ti.i64, ti.f32, ti.f64]
-test_dsize = [(4**i) * kibibyte for i in range(1, 11)]  #[4KB,16KB...1GB]
-test_repeat = 10
-results_evaluation = [geometric_mean]
 
+class SuiteInfo:
+    cases = [fill, saxpy, reduction]
+    supported_archs = [ti.x64, ti.cuda]
+    dtype = [ti.i32, ti.i64, ti.f32, ti.f64]
+    dsize = [(4**i) * kibibyte for i in range(1, 10)]  #[4KB,16KB...256MB]
+    repeat = 10
+    evaluator = [geometric_mean]
 
-class BenchmarkResult:
-    def __init__(self, name, arch, dtype, dsize, results_evaluation):
+
+class CaseResult:
+    def __init__(self, name, arch, dtype, dsize, evaluator):
         self.test_name = name
         self.test_arch = arch
         self.data_type = dtype
-        self.data_size = dsize
-        self.min_time_in_us = []
-        self.results_evaluation = results_evaluation
+        self.data_size = dsize  #list
+        self.min_time_in_us = []  #list
+        self.evaluator = evaluator
 
-    def time2mdtableline(self):
+    def result_to_markdown(self):
         string = '|' + self.test_name + '.' + dtype2str[self.data_type] + '|'
         string += ''.join(
             str(round(time, 4)) + '|' for time in self.min_time_in_us)
         string += ''.join(
             str(round(item(self.min_time_in_us), 4)) + '|'
-            for item in self.results_evaluation)
+            for item in self.evaluator)
         return string
 
-
-class BenchmarkImpl:
-    def __init__(self, func, archs, data_type, data_size):
+    def result_to_dict(self):
+        result_dicts = {}
+        for i in range(len(self.data_size)):
+            dsize = self.data_size[i]
+            reslut_name = size2str(dsize).replace('.0', '')
+            repeat = repeat_times(self.test_arch, dsize, SuiteInfo.repeat)
+            elapsed_time = self.min_time_in_us[i]
+            item_dict = {
+                'dsize_byte': dsize,
+                'repeat': repeat,
+                'elapsed_time_ms': elapsed_time
+            }
+            result_dicts[reslut_name] = item_dict
+        return result_dicts
+
+
+class CaseImpl:
+    def __init__(self, func, arch, data_type, data_size):
         self.func = func
         self.name = func.__name__
         self.env = None
         self.device = None
-        self.archs = archs
+        self.arch = arch
         self.data_type = data_type
         self.data_size = data_size
-        self.benchmark_results = []
+        self.case_results = []
 
     def run(self):
-        for arch in self.archs:
-            for dtype in self.data_type:
-                ti.init(kernel_profiler=True, arch=arch)
-                print("TestCase[%s.%s.%s]" %
-                      (self.func.__name__, ti.core.arch_name(arch),
-                       dtype2str[dtype]))
-                result = BenchmarkResult(self.name, arch, dtype,
-                                         self.data_size, results_evaluation)
-                for size in self.data_size:
-                    print("data_size = %s" % (size2str(size)))
-                    result.min_time_in_us.append(
-                        self.func(arch, dtype, size, test_repeat))
-                    time.sleep(0.2)
-                self.benchmark_results.append(result)
-
-    def print(self):
-        i = 0
-        for arch in self.archs:
-            for dtype in self.data_type:
-                for idx in range(len(self.data_size)):
-                    print(
-                        "    test_case:[%s] arch:[%s] dtype:[%s] dsize:[%7s] >>> time:[%4.4f]"
-                        %
-                        (self.name, ti.core.arch_name(arch), dtype2str[dtype],
-                         size2str(self.benchmark_results[i].data_size[idx]),
-                         self.benchmark_results[i].min_time_in_us[idx]))
-                i = i + 1
-
-    def save2markdown(self, arch):
+        for dtype in self.data_type:
+            ti.init(kernel_profiler=True, arch=self.arch)
+            print("TestCase[%s.%s.%s]" %
+                  (self.func.__name__, arch_name(self.arch), dtype2str[dtype]))
+            result = CaseResult(self.name, self.arch, dtype, self.data_size,
+                                SuiteInfo.evaluator)
+            for size in self.data_size:
+                print("data_size = %s" % (size2str(size)))
+                result.min_time_in_us.append(
+                    self.func(self.arch, dtype, size, SuiteInfo.repeat))
+                time.sleep(0.2)
+            self.case_results.append(result)
+
+    def to_markdown(self):
         header = '|kernel elapsed time(ms)' + ''.join(
-            '|' for i in range(len(self.data_size) + len(results_evaluation)))
+            '|' for i in range(len(self.data_size) + len(SuiteInfo.evaluator)))
         lines = [header]
-        for result in self.benchmark_results:
-            if (result.test_arch == arch):
-                lines.append(result.time2mdtableline())
+        for result in self.case_results:
+            lines.append(result.result_to_markdown())
         return lines
 
 
-class Membound:
-    benchmark_imps = []
+class MemoryBound:
+    suite_name = 'memorybound'
+    supported_archs = SuiteInfo.supported_archs
 
-    def __init__(self):
-        for case in test_cases:
-            self.benchmark_imps.append(
-                BenchmarkImpl(case, test_archs, test_dtype, test_dsize))
+    def __init__(self, arch):
+        self.arch = arch
+        self.cases_impl = []
+        for case in SuiteInfo.cases:
+            self.cases_impl.append(
+                CaseImpl(case, arch, SuiteInfo.dtype, SuiteInfo.dsize))
 
     def run(self):
-        for case in self.benchmark_imps:
+        for case in self.cases_impl:
             case.run()
 
-    def mdlines(self, arch):
+    def get_suite_info(self):
+        info_dict = {
+            'cases': [func.__name__ for func in SuiteInfo.cases],
+            'dtype': [dtype2str[type] for type in SuiteInfo.dtype],
+            'dsize': [size for size in SuiteInfo.dsize],
+            'repeat': [
+                repeat_times(self.arch, size, SuiteInfo.repeat)
+                for size in SuiteInfo.dsize
+            ],
+            'evaluator': [func.__name__ for func in SuiteInfo.evaluator]
+        }
+        return dump2json(info_dict)
+
+    def get_markdown_str(self):
         lines = []
-        lines += md_table_header(self.__class__.__name__, arch, test_dsize,
-                                 test_repeat, results_evaluation)
-        for case in self.benchmark_imps:
-            if arch in case.archs:
-                lines += case.save2markdown(arch)
-            else:
-                continue
+        lines += md_table_header(self.suite_name, self.arch, SuiteInfo.dsize,
+                                 SuiteInfo.repeat, SuiteInfo.evaluator)
+        for case in self.cases_impl:
+            lines += case.to_markdown()
         lines.append('')
         return lines
+
+    def save_to_json(self, suite_path='./'):
+        #save suite benchmark result to case.json
+        for case in self.cases_impl:  #[fill,saxpy,reduction]
+            case_path = os.path.join(suite_path, (case.name + '.json'))
+            results_dict = {}
+            for result in case.case_results:
+                type_str = dtype2str[result.data_type]
+                result_name = self.suite_name + '.' + case.name + '.' + arch_name(
+                    self.arch) + '.' + type_str
+                results_dict[result_name] = result.result_to_dict()
+            with open(case_path, 'w') as f:
+                case_str = dump2json(results_dict)
+                print(case_str, file=f)
diff --git a/benchmarks/misc/membound_cases.py b/benchmarks/misc/membound_cases.py
index b02b76382..1a44a671d 100644
--- a/benchmarks/misc/membound_cases.py
+++ b/benchmarks/misc/membound_cases.py
@@ -1,4 +1,4 @@
-from utils import dtype_size, scale_repeat, size2str
+from utils import dtype_size, repeat_times
 
 import taichi as ti
 
@@ -25,7 +25,7 @@ def membound_benchmark(func, num_elements, repeat):
 
 def fill(arch, dtype, dsize, repeat=10):
 
-    repeat = scale_repeat(arch, dsize, repeat)
+    repeat = repeat_times(arch, dsize, repeat)
     num_elements = dsize // dtype_size[dtype]
 
     x = ti.field(dtype, shape=num_elements)
@@ -40,7 +40,7 @@ def fill_const(n: ti.i32):
 
 def saxpy(arch, dtype, dsize, repeat=10):
 
-    repeat = scale_repeat(arch, dsize, repeat)
+    repeat = repeat_times(arch, dsize, repeat)
     num_elements = dsize // dtype_size[dtype] // 3  #z=x+y
 
     x = ti.field(dtype, shape=num_elements)
@@ -60,7 +60,7 @@ def saxpy(n: ti.i32):
 
 def reduction(arch, dtype, dsize, repeat=10):
 
-    repeat = scale_repeat(arch, dsize, repeat)
+    repeat = repeat_times(arch, dsize, repeat)
     num_elements = dsize // dtype_size[dtype]
 
     x = ti.field(dtype, shape=num_elements)
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index 69fff53d8..f3d7c2a55 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -1,33 +1,116 @@
-from membound import Membound
+import os
+
+from membound import MemoryBound
+from taichi.core import ti_core as _ti_core
+from utils import arch_name, datatime_with_format, dump2json
 
 import taichi as ti
 
-test_suites = [Membound]
-test_archs = [ti.cuda]
+benchmark_suites = [MemoryBound]
+benchmark_archs = [ti.cpu]  #, ti.cuda]
 
+#PR/id/
 
-class PerformanceMonitoring:
-    suites = []
 
-    def __init__(self):
-        for s in test_suites:
-            self.suites.append(s())
+class BenchmarkInfo:
+    def __init__(self, pull_request_id, commit_hash):
+        self.pull_request_id = pull_request_id  #int
+        self.commit_hash = commit_hash  #str
+        self.archs = []  #list ['x64','CUDA','Vulkan', ...]
+        self.datetime = []  #list [begin, end]
+
+
+class PerformanceMonitoring:
+    def __init__(self, arch):
+        self.suites = []
+        self.arch = arch
+        for suite in benchmark_suites:
+            self.suites.append(suite(arch))
 
     def run(self):
-        print("Running...")
+        print(f'Arch : {arch_name(self.arch)} Running...')
+        for suite in self.suites:
+            suite.run()
+
+    def save_to_json(self, file_dir='./'):
+        #arch info
+        arch_dict = {}
+        arch_dict['arch_name'] = arch_name(self.arch)
+        arch_dict['suites'] = [suite.suite_name for suite in self.suites]
+        info_path = os.path.join(file_dir, '_info.json')
+        info_str = dump2json(arch_dict)
+        with open(info_path, 'w') as f:
+            print(info_str, file=f)
+        #suite info
+        for suite in self.suites:
+            #suite folder
+            suite_path = os.path.join(file_dir, suite.suite_name)
+            os.makedirs(suite_path)
+            #suite info
+            info_path = os.path.join(suite_path, '_info.json')
+            info_str = suite.get_suite_info()
+            with open(info_path, 'w') as f:
+                print(info_str, file=f)
+            #cases info
+            suite.save_to_json(suite_path)
+
+    def save_to_markdown(self, arch_dir='./'):
+        current_time = datatime_with_format()
+        commit_hash = _ti_core.get_commit_hash()  #[:8]
         for s in self.suites:
-            s.run()
+            file_name = f'{s.suite_name}.md'
+            path = os.path.join(arch_dir, s.suite_name, file_name)
+            with open(path, 'w') as f:
+                lines = [
+                    f'commit_hash: {commit_hash}\n',
+                    f'datatime: {current_time}\n'
+                ] + s.get_markdown_str()
+                for line in lines:
+                    print(line, file=f)
+
+
+def check_supported(arch_list):
+    for arch in arch_list:
+        for suite in benchmark_suites:
+            if arch not in suite.supported_archs:
+                raise RuntimeError(
+                    'arch[' + arch_name(arch) +
+                    '] does not exist in SuiteInfo.supported_archs of class ' +
+                    suite.__name__)
+
+
+def main():
+    # TODO parser & test_archs = sys.argv[1]
+    test_archs = benchmark_archs  #default
+    check_supported(test_archs)
+
+    result_file_name = 'results'
+    benchmark_dir = os.path.join(os.getcwd(), result_file_name)
+    os.makedirs(benchmark_dir)
 
-    def write_md(self):
-        filename = f'performance_result.md'
-        with open(filename, 'w') as f:
-            for arch in test_archs:
-                for s in self.suites:
-                    lines = s.mdlines(arch)
-                    for line in lines:
-                        print(line, file=f)
+    pull_request_id = os.environ.get('PULL_REQUEST_ID')
+    commit_hash = _ti_core.get_commit_hash()  #[:8]
+    info = BenchmarkInfo(pull_request_id, commit_hash)
+    info.datetime.append(datatime_with_format())  #start time
+    for arch in test_archs:
+        #make dir
+        arch_dir = os.path.join(benchmark_dir, arch_name(arch))
+        os.makedirs(arch_dir)
+        #init & run
+        impl = PerformanceMonitoring(arch)
+        impl.run()
+        #append info
+        info.archs.append(arch_name(arch))
+        #save result
+        impl.save_to_json(arch_dir)
+        impl.save_to_markdown(arch_dir)
+    info.datetime.append(datatime_with_format())  #end time
+    #save benchmark info
+    info_path = os.path.join(benchmark_dir, '_info.json')
+    info_str = dump2json(info)
+    with open(info_path, 'w') as f:
+        print(info_str, file=f)
 
 
-p = PerformanceMonitoring()
-p.run()
-p.write_md()
+if __name__ == '__main__':
+    main()
diff --git a/benchmarks/misc/utils.py b/benchmarks/misc/utils.py
index 5e6206116..2debe1966 100644
--- a/benchmarks/misc/utils.py
+++ b/benchmarks/misc/utils.py
@@ -1,3 +1,9 @@
+import datetime
+import json
+
+import jsbeautifier
+from taichi.core import ti_core as _ti_core
+
 import taichi as ti
 
 kibibyte = 1024
@@ -13,21 +19,30 @@
                    (1073741824.0, 'GB'), (float('inf'), 'INF')]  #B KB MB GB
 
 
+def arch_name(arch):
+    return _ti_core.arch_name(arch)
+
+
+def dump2json(obj):
+    if type(obj) is dict:
+        obj2dict = obj
+    else:
+        obj2dict = obj.__dict__
+    options = jsbeautifier.default_options()
+    options.indent_size = 4
+    return jsbeautifier.beautify(json.dumps(obj2dict), options)
+
+
+def datatime_with_format():
+    return datetime.datetime.now().isoformat()
+
+
 def size2str(size_in_byte):
     for dsize, units in reversed(size_subsection):
         if size_in_byte >= dsize:
             return str(round(size_in_byte / dsize, 4)) + units
 
 
-def scale_repeat(arch, datasize, repeat=10):
-    scaled = repeat
-    if (arch == ti.gpu) | (arch == ti.opengl) | (arch == ti.cuda):
-        scaled *= 10
-    if datasize <= 4 * 1024 * 1024:
-        scaled *= 10
-    return scaled
-
-
 def geometric_mean(data_array):
     product = 1
     for data in data_array:
@@ -35,6 +50,14 @@ def geometric_mean(data_array):
     return pow(product, 1.0 / len(data_array))
 
 
+def repeat_times(arch, datasize, repeat=1):
+    if (arch == ti.gpu) | (arch == ti.opengl) | (arch == ti.cuda):
+        repeat *= 10
+    if datasize <= 4 * 1024 * 1024:
+        repeat *= 10
+    return repeat
+
+
 def md_table_header(suite_name, arch, test_dsize, test_repeat,
                     results_evaluation):
     header = '|' + suite_name + '.' + ti.core.arch_name(arch) + '|'
@@ -51,7 +74,7 @@ def md_table_header(suite_name, arch, test_dsize, test_repeat,
 
     repeat = '|**repeat**|'
     repeat += ''.join(
-        str(scale_repeat(arch, size, test_repeat)) + '|'
+        str(repeat_times(arch, size, test_repeat)) + '|'
         for size in test_dsize)
     repeat += ''.join('|' for i in range(len(results_evaluation)))
 

From ffd8e0b37fa8207c4763b584156f147d8c8ca5d5 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 25 Oct 2021 23:11:55 +0800
Subject: [PATCH 04/24] wip

---
 .github/workflows/presubmit.yml | 2 +-
 benchmarks/misc/run.py          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 252a9123a..c7702df12 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -91,7 +91,7 @@ jobs:
           echo $BENCHMARK_CODE_DIR
           cd ..
           # update benchmark code
-          git cloen git@github.com:yolo2themoon/perf_monitoring.git
+          git clone git@github.com:yolo2themoon/perf_monitoring.git
           cd perf_monitoring
           export PERF_MONITORING_REPO_DIR=`pwd`
           echo $PERF_MONITORING_REPO_DIR
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index f3d7c2a55..dd3c9658f 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -9,7 +9,7 @@
 benchmark_suites = [MemoryBound]
 benchmark_archs = [ti.cpu]  #, ti.cuda]
 
-#PR/id/
+#PR/id/1234
 
 
 class BenchmarkInfo:

From 7e9eb3421f1c7ba4634138108d4ec92a7fbb55f9 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 25 Oct 2021 23:36:20 +0800
Subject: [PATCH 05/24] wip

---
 .github/workflows/presubmit.yml | 14 ++++++--------
 benchmarks/misc/run.py          |  3 ++-
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index c7702df12..2487d7a51 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -85,16 +85,14 @@ jobs:
       - name: Run benchmark
         run: |
           # in Taichi dir
-          export TAICHI_REPO_DIR=`pwd`
-          export BENCHMARK_CODE_DIR=$TAICHI_REPO_DIR/benchmarks/misc/
-          echo $TAICHI_REPO_DIR
-          echo $BENCHMARK_CODE_DIR
+          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
           cd ..
-          # update benchmark code
+          # pull repo : performance-monitoring
           git clone git@github.com:yolo2themoon/perf_monitoring.git
           cd perf_monitoring
-          export PERF_MONITORING_REPO_DIR=`pwd`
-          echo $PERF_MONITORING_REPO_DIR
-          cp -r $BENCHMARK_CODE_DIR/* $PERF_MONITORING_REPO_DIR/benchmarks/
+          # update benchmark code
+          cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
           # run benchmark
+          export PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')
+          echo $PULL_REQUEST_NUMBER
           ./run.sh
\ No newline at end of file
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index dd3c9658f..e3544f3f2 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -88,7 +88,8 @@ def main():
     benchmark_dir = os.path.join(os.getcwd(), result_file_name)
     os.makedirs(benchmark_dir)
 
-    pull_request_id = os.environ.get('PULL_REQUEST_ID')
+    pull_request_id = os.environ.get('PULL_REQUEST_NUMBER')
+    print(f'==========={pull_request_id}=============')
     commit_hash = _ti_core.get_commit_hash()  #[:8]
     info = BenchmarkInfo(pull_request_id, commit_hash)
     info.datetime.append(datatime_with_format())  #start time

From 7c02050a90ceaca6d2fbcd1211c631053f821be5 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 25 Oct 2021 23:43:30 +0800
Subject: [PATCH 06/24] wip

---
 .github/workflows/presubmit.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 2487d7a51..95445cb1d 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -84,15 +84,18 @@ jobs:
 
       - name: Run benchmark
         run: |
+          export PULL_REQUEST_NUMBER=${{github.event.issue.number}}
+          echo $PULL_REQUEST_NUMBER
+          export X_NUMBER=${{github.event.number}}
+          echo $X_NUMBER
           # in Taichi dir
           export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
           cd ..
           # pull repo : performance-monitoring
+          rm -rf perf_monitoring
           git clone git@github.com:yolo2themoon/perf_monitoring.git
           cd perf_monitoring
           # update benchmark code
           cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
           # run benchmark
-          export PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')
-          echo $PULL_REQUEST_NUMBER
           ./run.sh
\ No newline at end of file

From 91423295dc5f77639ae89d41bdd63682a9914cdb Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 25 Oct 2021 23:52:07 +0800
Subject: [PATCH 07/24] wip

---
 .github/workflows/presubmit.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 95445cb1d..513e2e5ea 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -84,10 +84,7 @@ jobs:
 
       - name: Run benchmark
         run: |
-          export PULL_REQUEST_NUMBER=${{github.event.issue.number}}
-          echo $PULL_REQUEST_NUMBER
-          export X_NUMBER=${{github.event.number}}
-          echo $X_NUMBER
+          export PULL_REQUEST_NUMBER=${{github.event.number}}
           # in Taichi dir
           export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
           cd ..

From 0aef1776011c08cd4e4d2c3812ef11d9ea59fa5c Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Tue, 26 Oct 2021 00:04:20 +0800
Subject: [PATCH 08/24] wip

---
 .github/workflows/presubmit.yml | 24 +++++++++++++++---------
 benchmarks/misc/run.py          |  2 +-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 513e2e5ea..5b9fa6a7e 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -73,6 +73,8 @@ jobs:
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
           .github/workflows/scripts/unix_build.sh
+          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
+          cd ..
         env:
           LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
           LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
@@ -82,17 +84,21 @@ jobs:
           BUILD_NUM_THREADS: 8
           CXX: clang++-10
 
-      - name: Run benchmark
+      - name: Clone repo performance-monitoring
         run: |
-          export PULL_REQUEST_NUMBER=${{github.event.number}}
-          # in Taichi dir
-          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
-          cd ..
-          # pull repo : performance-monitoring
           rm -rf perf_monitoring
           git clone git@github.com:yolo2themoon/perf_monitoring.git
           cd perf_monitoring
-          # update benchmark code
+
+      - name: Update repo (benchmarks code)
+        run: |
           cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
-          # run benchmark
-          ./run.sh
\ No newline at end of file
+
+      - name: Run benchmark & store result
+        run: |
+          export PULL_REQUEST_NUMBER=${{github.event.number}}
+          ./run.sh
+
+      - name: Update repo (git push reslut)
+        run: |
+          ./scripts/push.sh
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index e3544f3f2..910b99199 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -89,7 +89,7 @@ def main():
     os.makedirs(benchmark_dir)
 
     pull_request_id = os.environ.get('PULL_REQUEST_NUMBER')
-    print(f'==========={pull_request_id}=============')
+    print(f'pull_request_id = {pull_request_id}')
     commit_hash = _ti_core.get_commit_hash()  #[:8]
     info = BenchmarkInfo(pull_request_id, commit_hash)
     info.datetime.append(datatime_with_format())  #start time

From d2f1006071e43f9401ad7a39ec902e1cc175cc1a Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Tue, 26 Oct 2021 00:20:11 +0800
Subject: [PATCH 09/24] wip

---
 .github/workflows/presubmit.yml | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 5b9fa6a7e..58ed389bc 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -73,8 +73,6 @@ jobs:
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
           .github/workflows/scripts/unix_build.sh
-          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
-          cd ..
         env:
           LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
           LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
@@ -84,21 +82,19 @@ jobs:
           BUILD_NUM_THREADS: 8
           CXX: clang++-10
 
-      - name: Clone repo performance-monitoring
+      - name: Run performance-monitoring
         run: |
+          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
+          cd ..
+          # clone repo
           rm -rf perf_monitoring
           git clone git@github.com:yolo2themoon/perf_monitoring.git
           cd perf_monitoring
-
-      - name: Update repo (benchmarks code)
-        run: |
+          # update repo (benchmarks code)
           cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
-
-      - name: Run benchmark & store result
-        run: |
-          export PULL_REQUEST_NUMBER=${{github.event.number}}
+          # run benchmark & store result
           ./run.sh
-
-      - name: Update repo (git push reslut)
-        run: |
+          # update repo (push reslut)
           ./scripts/push.sh
+        env:
+          PULL_REQUEST_NUMBER: ${{github.event.number}}

From ee3520d7d039fa65c157534d26742b5dda2af688 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Tue, 26 Oct 2021 00:40:54 +0800
Subject: [PATCH 10/24] test performance-monitoring

---
 .github/workflows/presubmit.yml | 6 +++---
 benchmarks/misc/run.py          | 4 +---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 58ed389bc..064113050 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -87,9 +87,9 @@ jobs:
           export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
           cd ..
           # clone repo
-          rm -rf perf_monitoring
-          git clone git@github.com:yolo2themoon/perf_monitoring.git
-          cd perf_monitoring
+          rm -rf performance-monitoring
+          git clone git@github.com:taichi-dev/performance-monitoring.git
+          cd performance-monitoring
           # update repo (benchmarks code)
           cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
           # run benchmark & store result
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index 910b99199..e870a0176 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -7,9 +7,7 @@
 import taichi as ti
 
 benchmark_suites = [MemoryBound]
-benchmark_archs = [ti.cpu]  #, ti.cuda]
-
-#PR/id/1234
+benchmark_archs = [ti.cpu, ti.cuda]
 
 
 class BenchmarkInfo:

From e795b16e7f122c026e2d0b008edc3844b4df1f02 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Mon, 1 Nov 2021 12:09:52 +0800
Subject: [PATCH 11/24] [CI] Linux CD containerization (#107)

---
 .github/workflows/release.yml                 | 78 ++++++++-----------
 .../workflows/scripts/unix_docker_build.sh    | 32 ++++++++
 .github/workflows/scripts/unix_docker_test.sh | 29 +++++++
 .github/workflows/scripts/unix_test.sh        |  1 +
 4 files changed, 96 insertions(+), 44 deletions(-)
 create mode 100755 .github/workflows/scripts/unix_docker_build.sh
 create mode 100755 .github/workflows/scripts/unix_docker_test.sh

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 33a82fea0..7e25da284 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -20,9 +20,9 @@ jobs:
       - id: set-matrix
         run: |
           # For nightly release, we only run on python 3.8
-          [ -z "${{ github.event.action }}" ] && matrix="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\"}]"
+          [ -z "${{ github.event.action }}" ] && matrix="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\",\"conda_python\":\"py38\"}]"
           # For production release, we run on four python versions.
-          [ -z "${{ github.event.action }}" ] || matrix="[{\"name\":\"taichi\",\"python\":\"3.6\"},{\"name\":\"taichi\",\"python\":\"3.7\"},{\"name\":\"taichi\",\"python\":\"3.8\"},{\"name\":\"taichi\",\"python\":\"3.9\"}]"
+          [ -z "${{ github.event.action }}" ] || matrix="[{\"name\":\"taichi\",\"python\":\"3.6\",\"conda_python\":\"py36\"},{\"name\":\"taichi\",\"python\":\"3.7\",\"conda_python\":\"py37\"},{\"name\":\"taichi\",\"python\":\"3.8\",\"conda_python\":\"py38\"},{\"name\":\"taichi\",\"python\":\"3.9\",\"conda_python\":\"py39\"}]"
           echo ::set-output name=matrix::{\"include\":$(echo $matrix)}\"
           # M1 only supports py38 and py39(conda), so change matrix.
           [ -z "${{ github.event.action }}" ] && matrix_osx="[{\"name\":\"taichi-nightly\",\"python\":\"3.8\"}]"
@@ -41,56 +41,36 @@ jobs:
         with:
           submodules: "recursive"
 
-      - name: Create Python Wheel
+      - name: Build
         run: |
-          # We hacked here because conda activate in CI won't update python PATH
-          # automatically. So we don't activate and use desired python version
-          # directly.
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
-          TAICHI_REPO_DIR=`pwd`
-          export PATH=$LLVM_LIB_ROOT_DIR/bin:/usr/local/cuda/bin:$PATH
-          export LLVM_DIR=$LLVM_LIB_ROOT_DIR/lib/cmake/llvm
-          export CXX=clang++-8
-          python3 -m pip uninstall taichi taichi-nightly -y
-          python3 -m pip install -r requirements_dev.txt
-          python3 -m pip install twine
-          cd python
-          git fetch origin master
-          export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
-          python3 build.py build --project_name $PROJECT_NAME
-          cd ..
-          NUM_WHL=`ls dist/*.whl | wc -l`
-          if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
-          python3 -m pip install dist/*.whl
+          mkdir -m777 wheel
+          docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
+          tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/
+          docker start -a taichi_build
         env:
-          LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
-          BUILD_NUM_THREADS: 8
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
+          PY: ${{ matrix.conda_python }}
+          GPU_BUILD: ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
-          PYTHON: ${{ matrix.python }}
+          DISPLAY: :1
 
       - name: Archive Wheel Artifacts
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.name }}-py${{ matrix.python }}-linux.whl
-          path: dist/*.whl
+          path: wheel/*.whl
           retention-days: 20
 
       - name: Test
         run: |
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
-          python3 examples/algorithm/laplace.py
-          export DISPLAY=:1
-          hash -r
-          glewinfo
-          ti diagnose
-          ti changelog
-          ti test -vr2 -t2 -k "not ndarray and not torch"
-          # ndarray test might OOM if run with -t2.
-          # FIXME: unify this with presubmit.yml to avoid further divergence
-          ti test -vr2 -t1 -k "ndarray or torch"
+          docker create --user dev --name taichi_test --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/unix_docker_test.sh $PY $GPU_TEST
+          docker cp .github/workflows/scripts/unix_docker_test.sh taichi_test:/home/dev/unix_docker_test.sh
+          docker cp wheel/*.whl taichi_test:/home/dev/
+          docker start -a taichi_test
         env:
-          PYTHON: ${{ matrix.python }}
+          PY: py38
+          GPU_TEST: ON
+          DISPLAY: :1
 
       - name: Upload PyPI
         env:
@@ -100,11 +80,17 @@ jobs:
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
         run: |
-          export PATH=/home/buildbot/miniconda3/envs/$PYTHON/bin:$PATH
+          mkdir dist
+          cp wheel/*.whl dist/
           cd python
           if [ $PROJECT_NAME == "taichi-nightly" ]; then export PYPI_PWD="$NIGHT_PWD" && python3 build.py upload --skip_build --testpypi --project_name $PROJECT_NAME
           elif [ $PROJECT_NAME == "taichi" ]; then export PYPI_PWD="$PROD_PWD" && python3 build.py upload --skip_build; fi
 
+      - name: clean docker container
+        if: always()
+        run: |
+          docker rm taichi_build taichi_test -f
+
   build_and_upload_mac:
     name: Build and Upload (macOS only)
     needs: matrix_prep
@@ -132,6 +118,7 @@ jobs:
           export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
           export CXX=clang++
           python -m pip install -r requirements_dev.txt
+          python -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -141,7 +128,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
           PROJECT_NAME: ${{ matrix.name }}
 
       - name: Archive Wheel Artifacts
@@ -155,7 +142,7 @@ jobs:
         run: |
           python examples/algorithm/laplace.py
           ti diagnose
-          ti test -vr2 -t2
+          ti test -vr2 -t2 -a cpu,metal
 
       - name: Upload PyPI
         env:
@@ -193,6 +180,7 @@ jobs:
           git --version
           export CXX=clang++
           python3 -m pip install -r requirements_dev.txt
+          python3 -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -202,7 +190,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           python3 -m pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
 
@@ -250,6 +238,7 @@ jobs:
           git --version
           export CXX=clang++
           python3 -m pip install -r requirements_dev.txt
+          python3 -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           export TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS
@@ -259,7 +248,7 @@ jobs:
           if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
           python3 -m pip install dist/*.whl
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_TESTS:BOOL=ON
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_TESTS:BOOL=ON
           PROJECT_NAME: ${{ matrix.name }}
           PYTHON: ${{ matrix.python }}
 
@@ -336,6 +325,7 @@ jobs:
           clang --version
           cd D:\a\taichi\taichi
           python -m pip install -r requirements_dev.txt
+          python -m pip install -r requirements_test.txt
           cd python
           git fetch origin master
           $env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh
new file mode 100755
index 000000000..cb7fbf05e
--- /dev/null
+++ b/.github/workflows/scripts/unix_docker_build.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+PY=$1
+GPU_BUILD=$2
+PROJECT_NAME=$3
+CI_SETUP_CMAKE_ARGS=$4
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+python3 -m pip uninstall taichi taichi-nightly -y
+
+cd taichi
+
+if [[ $GPU_BUILD == "OFF" ]]
+then 
+    python3 -m pip install -r requirements_dev.txt
+fi
+
+cd python
+# This is for changelog
+git fetch origin master
+TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build --project_name $PEOJECT_NAME
+# Run basic cpp tests
+cd ..
+CUR_DIR=`pwd`
+TI_LIB_DIR=$CUR_DIR/python/taichi/lib ./build/taichi_cpp_tests
+
+cp dist/*.whl /wheel/
diff --git a/.github/workflows/scripts/unix_docker_test.sh b/.github/workflows/scripts/unix_docker_test.sh
new file mode 100755
index 000000000..d3df0c2b5
--- /dev/null
+++ b/.github/workflows/scripts/unix_docker_test.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+PY=$1
+GPU_TEST=$2
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+python3 -m pip install ./*.whl
+
+if [[ $GPU_TEST == "OFF" ]]
+then
+    python3 -m pip install -r requirements_test.txt
+fi
+
+export TI_IN_DOCKER=true
+ti diagnose
+ti changelog
+
+if [[ $GPU_TEST == "OFF" ]]
+then
+    ti test -vr2 -t2
+else
+    ti test -vr2 -t2 -k "not ndarray and not torch"
+    ti test -vr2 -t1 -k "ndarray or torch"
+fi
diff --git a/.github/workflows/scripts/unix_test.sh b/.github/workflows/scripts/unix_test.sh
index 907c52160..53bf5d07e 100755
--- a/.github/workflows/scripts/unix_test.sh
+++ b/.github/workflows/scripts/unix_test.sh
@@ -1,5 +1,6 @@
 set -ex
 TAICHI_REPO_DIR=`pwd`
+python3 -m pip install -r requirements_test.txt
 TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
 [[ $RUN_CPP_TESTS == "ON" ]] && TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
 export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH

From 780c479e1b0a4cc6d558a2f8773efe882e9f3680 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Mon, 1 Nov 2021 14:21:06 +0800
Subject: [PATCH 12/24] [CI] Containerize CD (#108)

* [CI] Linux CD containerization

* [CI] Linux CD containerization
---
 .github/workflows/release.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7e25da284..92b51dea8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -44,7 +44,8 @@ jobs:
       - name: Build
         run: |
           mkdir -m777 wheel
-          docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/taichi/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
+
+          docker create -v `pwd`/wheel:/wheel --user dev --name taichi_build --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix registry.taichigraphics.com/taichidev-ubuntu18.04:v0.1.1 /home/dev/test_actions/.github/workflows/scripts/unix_docker_build.sh $PY $GPU_BUILD $PROJECT_NAME "$CI_SETUP_CMAKE_ARGS"
           tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner dev --group dev | docker cp - taichi_build:/home/dev/
           docker start -a taichi_build
         env:

From 13a5bbbc0e61550cb68e6eed1acfb00924530b97 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Mon, 1 Nov 2021 14:28:06 +0800
Subject: [PATCH 13/24] Update unix_docker_build.sh

---
 .github/workflows/scripts/unix_docker_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh
index cb7fbf05e..d4a9ece40 100755
--- a/.github/workflows/scripts/unix_docker_build.sh
+++ b/.github/workflows/scripts/unix_docker_build.sh
@@ -13,7 +13,7 @@ conda activate $PY
 
 python3 -m pip uninstall taichi taichi-nightly -y
 
-cd taichi
+cd test_actions
 
 if [[ $GPU_BUILD == "OFF" ]]
 then 

From b9595fd3ff54d277800a0e47b3a44f3bb6a13b44 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Mon, 1 Nov 2021 14:32:10 +0800
Subject: [PATCH 14/24] Update unix_docker_build.sh

---
 .github/workflows/scripts/unix_docker_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scripts/unix_docker_build.sh b/.github/workflows/scripts/unix_docker_build.sh
index d4a9ece40..7868ecc88 100755
--- a/.github/workflows/scripts/unix_docker_build.sh
+++ b/.github/workflows/scripts/unix_docker_build.sh
@@ -23,7 +23,7 @@ fi
 cd python
 # This is for changelog
 git fetch origin master
-TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build --project_name $PEOJECT_NAME
+TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build --project_name $PROJECT_NAME
 # Run basic cpp tests
 cd ..
 CUR_DIR=`pwd`

From accdb225239cfb7bb5ad851f0f1cf29f89cd35f4 Mon Sep 17 00:00:00 2001
From: Jiasheng Zhang <leonz5288@hotmail.com>
Date: Mon, 1 Nov 2021 14:41:59 +0800
Subject: [PATCH 15/24] Update release.yml

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 92b51dea8..03956a923 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -69,7 +69,7 @@ jobs:
           docker cp wheel/*.whl taichi_test:/home/dev/
           docker start -a taichi_test
         env:
-          PY: py38
+          PY: ${{ matrix.conda_python }}
           GPU_TEST: ON
           DISPLAY: :1
 

From da1679d37b4201c0797ff93cfdfaef1b930921a7 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 16:55:22 +0800
Subject: [PATCH 16/24] context

---
 .github/workflows/presubmit.yml | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 064113050..f41addac7 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -81,20 +81,8 @@ jobs:
           CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON
           BUILD_NUM_THREADS: 8
           CXX: clang++-10
-
-      - name: Run performance-monitoring
-        run: |
-          export BENCHMARK_CODE_DIR=`pwd`/benchmarks/misc/
-          cd ..
-          # clone repo
-          rm -rf performance-monitoring
-          git clone git@github.com:taichi-dev/performance-monitoring.git
-          cd performance-monitoring
-          # update repo (benchmarks code)
-          cp -r $BENCHMARK_CODE_DIR/* ./benchmarks/
-          # run benchmark & store result
-          ./run.sh
-          # update repo (push reslut)
-          ./scripts/push.sh
+      
+      - name: Dump GitHub context
         env:
-          PULL_REQUEST_NUMBER: ${{github.event.number}}
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"

From ca8c02aadb03d52a38461b27e55036056acd9361 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 17:16:44 +0800
Subject: [PATCH 17/24] update

---
 .github/workflows/presubmit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 60bd1b361..49ec4f17a 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -62,7 +62,7 @@ jobs:
   
   performance_monitoring:
     name: Performance monitoring (NVGPU)
-    needs: check_code_format
+    needs: title_format
     timeout-minutes: 60
     runs-on: [self-hosted, x64, cuda, linux, benchmark]
     steps:

From 3bc15a297e04222cd44cd91e13c573b0f778fbf7 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 17:43:12 +0800
Subject: [PATCH 18/24] x

---
 .github/workflows/presubmit.yml | 17 ++++++++++++++--
 benchmarks/misc/test.py         | 35 +++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 benchmarks/misc/test.py

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index 49ec4f17a..e2a098c23 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -62,7 +62,6 @@ jobs:
   
   performance_monitoring:
     name: Performance monitoring (NVGPU)
-    needs: title_format
     timeout-minutes: 60
     runs-on: [self-hosted, x64, cuda, linux, benchmark]
     steps:
@@ -70,7 +69,21 @@ jobs:
         with:
           submodules: 'recursive'
 
+      - name: Build & Install
+        run: |
+          export PATH=$PATH:/usr/local/cuda/bin
+          .github/workflows/scripts/unix_build.sh
+        env:
+          LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
+          LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
+          LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
+          CUDA_TOOLKIT_ROOT_DIR: /usr/local/cuda/
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON
+          BUILD_NUM_THREADS: 8
+          CXX: clang++-10
+
       - name: Dump GitHub context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
+        run: |
+          python3 benchmarks/misc/test.py
diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
new file mode 100644
index 000000000..aabb85b43
--- /dev/null
+++ b/benchmarks/misc/test.py
@@ -0,0 +1,35 @@
+
+import sys
+import os
+import json
+import jsbeautifier
+import datetime
+import re
+
+perf_monitoring_dir = os.environ.get('PERF_MONITORING_DIR')
+if perf_monitoring_dir is None:
+    raise RuntimeError('Missing environment variable PERF_MONITORING_DIR')
+
+def dump2json(obj):
+    if type(obj) is dict:
+        obj2dict = obj
+    else:
+        obj2dict = obj.__dict__
+    options = jsbeautifier.default_options()
+    options.indent_size = 4
+    return jsbeautifier.beautify(json.dumps(obj2dict), options)
+
+def main():
+    github_context = os.environ.get('GITHUB_CONTEXT')
+    # info_path = os.path.join(perf_monitoring_dir, 'context.json')
+    #traverse pr_results
+    pr_dict = {}
+    # with open(info_path, 'r') as f:
+    pr_dict = json.loads(github_context)
+    pr_number = re.sub("\D","",pr_dict["ref"])
+    print(pr_dict["ref"])
+    print(pr_number)
+    # print(dump2json(pr_dict))
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file

From 9a18e1da0cd6a6290ba4c1095eaacf2f38a07050 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 17:47:50 +0800
Subject: [PATCH 19/24] x

---
 benchmarks/misc/test.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
index aabb85b43..871eede6f 100644
--- a/benchmarks/misc/test.py
+++ b/benchmarks/misc/test.py
@@ -6,10 +6,6 @@
 import datetime
 import re
 
-perf_monitoring_dir = os.environ.get('PERF_MONITORING_DIR')
-if perf_monitoring_dir is None:
-    raise RuntimeError('Missing environment variable PERF_MONITORING_DIR')
-
 def dump2json(obj):
     if type(obj) is dict:
         obj2dict = obj

From fb992085ff7a2cc9f44535a6db590e0bef3fa998 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 17:57:47 +0800
Subject: [PATCH 20/24] x

---
 benchmarks/misc/test.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
index 871eede6f..a479bde24 100644
--- a/benchmarks/misc/test.py
+++ b/benchmarks/misc/test.py
@@ -17,15 +17,11 @@ def dump2json(obj):
 
 def main():
     github_context = os.environ.get('GITHUB_CONTEXT')
-    # info_path = os.path.join(perf_monitoring_dir, 'context.json')
-    #traverse pr_results
-    pr_dict = {}
-    # with open(info_path, 'r') as f:
-    pr_dict = json.loads(github_context)
-    pr_number = re.sub("\D","",pr_dict["ref"])
-    print(pr_dict["ref"])
+    github_info_dict = {}
+    github_info_dict = json.loads(github_context)
+    pr_number = re.sub("\D","",github_info_dict["ref"])
+    print(github_info_dict["ref"])
     print(pr_number)
-    # print(dump2json(pr_dict))
 
 if __name__ == '__main__':
     main()
\ No newline at end of file

From a285bb3eba64ea7e17cc14169a7d339d91c42774 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 18:16:47 +0800
Subject: [PATCH 21/24] x

---
 benchmarks/misc/print.py | 15 +++++++++++++++
 benchmarks/misc/test.py  | 15 ++-------------
 2 files changed, 17 insertions(+), 13 deletions(-)
 create mode 100644 benchmarks/misc/print.py

diff --git a/benchmarks/misc/print.py b/benchmarks/misc/print.py
new file mode 100644
index 000000000..beb45451e
--- /dev/null
+++ b/benchmarks/misc/print.py
@@ -0,0 +1,15 @@
+
+
+
+
+import sys
+import os
+import json
+import re
+
+def main():
+    num = os.environ.get('PULL_REQUEST_NUMBER')
+    print(f'PULL_REQUEST_NUMBER = {num}')
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
index a479bde24..1b35cc21a 100644
--- a/benchmarks/misc/test.py
+++ b/benchmarks/misc/test.py
@@ -2,26 +2,15 @@
 import sys
 import os
 import json
-import jsbeautifier
-import datetime
 import re
 
-def dump2json(obj):
-    if type(obj) is dict:
-        obj2dict = obj
-    else:
-        obj2dict = obj.__dict__
-    options = jsbeautifier.default_options()
-    options.indent_size = 4
-    return jsbeautifier.beautify(json.dumps(obj2dict), options)
-
 def main():
     github_context = os.environ.get('GITHUB_CONTEXT')
     github_info_dict = {}
     github_info_dict = json.loads(github_context)
     pr_number = re.sub("\D","",github_info_dict["ref"])
-    print(github_info_dict["ref"])
-    print(pr_number)
+    os.system(f'export PULL_REQUEST_NUMBER={pr_number}')
+    print(f'pr_number = {pr_number}')
 
 if __name__ == '__main__':
     main()
\ No newline at end of file

From 49757dfb44d2750dc99599af33f8ba6b0400dd11 Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 18:25:25 +0800
Subject: [PATCH 22/24] a

---
 .github/workflows/presubmit.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
index e2a098c23..0b48dbe58 100644
--- a/.github/workflows/presubmit.yml
+++ b/.github/workflows/presubmit.yml
@@ -87,3 +87,4 @@ jobs:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: |
           python3 benchmarks/misc/test.py
+          python3 benchmarks/misc/print.py
\ No newline at end of file

From 7f678afac83df2154ff0d5a4882a47a7af60569d Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 18:31:04 +0800
Subject: [PATCH 23/24] a

---
 benchmarks/misc/test.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
index 1b35cc21a..679a73cfe 100644
--- a/benchmarks/misc/test.py
+++ b/benchmarks/misc/test.py
@@ -9,8 +9,9 @@ def main():
     github_info_dict = {}
     github_info_dict = json.loads(github_context)
     pr_number = re.sub("\D","",github_info_dict["ref"])
-    os.system(f'export PULL_REQUEST_NUMBER={pr_number}')
-    print(f'pr_number = {pr_number}')
+    set_pr_num_str = f'export PULL_REQUEST_NUMBER={pr_number}'
+    print(set_pr_num_str)
+    os.system(set_pr_num_str)
 
 if __name__ == '__main__':
     main()
\ No newline at end of file

From cebc5ed34756e34e218ef11b3561af27c9c7836c Mon Sep 17 00:00:00 2001
From: yolo2themoon <markbench@163.com>
Date: Mon, 1 Nov 2021 18:37:19 +0800
Subject: [PATCH 24/24] a

---
 benchmarks/misc/membound.py       |  1 +
 benchmarks/misc/membound_cases.py |  1 +
 benchmarks/misc/print.py          | 12 +++++-------
 benchmarks/misc/run.py            |  1 +
 benchmarks/misc/test.py           | 16 ++++++++--------
 benchmarks/misc/utils.py          |  1 +
 6 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/benchmarks/misc/membound.py b/benchmarks/misc/membound.py
index e69de29bb..8b1378917 100644
--- a/benchmarks/misc/membound.py
+++ b/benchmarks/misc/membound.py
@@ -0,0 +1 @@
+
diff --git a/benchmarks/misc/membound_cases.py b/benchmarks/misc/membound_cases.py
index e69de29bb..8b1378917 100644
--- a/benchmarks/misc/membound_cases.py
+++ b/benchmarks/misc/membound_cases.py
@@ -0,0 +1 @@
+
diff --git a/benchmarks/misc/print.py b/benchmarks/misc/print.py
index beb45451e..8b5c01809 100644
--- a/benchmarks/misc/print.py
+++ b/benchmarks/misc/print.py
@@ -1,15 +1,13 @@
-
-
-
-
-import sys
-import os
 import json
+import os
 import re
+import sys
+
 
 def main():
     num = os.environ.get('PULL_REQUEST_NUMBER')
     print(f'PULL_REQUEST_NUMBER = {num}')
 
+
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py
index e69de29bb..8b1378917 100644
--- a/benchmarks/misc/run.py
+++ b/benchmarks/misc/run.py
@@ -0,0 +1 @@
+
diff --git a/benchmarks/misc/test.py b/benchmarks/misc/test.py
index 679a73cfe..7f550176b 100644
--- a/benchmarks/misc/test.py
+++ b/benchmarks/misc/test.py
@@ -1,17 +1,17 @@
-
-import sys
-import os
 import json
+import os
 import re
+import sys
+
 
 def main():
     github_context = os.environ.get('GITHUB_CONTEXT')
     github_info_dict = {}
     github_info_dict = json.loads(github_context)
-    pr_number = re.sub("\D","",github_info_dict["ref"])
-    set_pr_num_str = f'export PULL_REQUEST_NUMBER={pr_number}'
-    print(set_pr_num_str)
-    os.system(set_pr_num_str)
+    pr_number = re.sub("\D", "", github_info_dict["ref"])
+    os.environ['PULL_REQUEST_NUMBER'] = pr_number
+    print(os.environ.get('PULL_REQUEST_NUMBER'))
+
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/benchmarks/misc/utils.py b/benchmarks/misc/utils.py
index e69de29bb..8b1378917 100644
--- a/benchmarks/misc/utils.py
+++ b/benchmarks/misc/utils.py
@@ -0,0 +1 @@
+