[benchmark] Store the benchmark results as json files (#3294)

* [benchmark] Store the benchmark results as json files for performance monitoring

* update

* add ti.cuda to suite.supported_archs

* rebase and trigger ci

benchmarks/misc/membound.py

@@ -1,78 +1,147 @@
+import os
 import time
 
 from membound_cases import memory_bound_cases_list
-from utils import (arch_name, dtype2str, geometric_mean, kibibyte,
-                   md_table_header, size2str)
+from utils import (arch_name, datatime_with_format, dtype2str, dump2json,
+                   geometric_mean, md_table_header, scaled_repeat_times,
+                   size2str)
 
 import taichi as ti
 
 
 class MemoryBound:
     suite_name = 'memorybound'
-    supported_archs = [ti.cpu, ti.cuda]
+    supported_archs = [ti.x64, ti.cuda]
     test_cases = memory_bound_cases_list
     test_dtype_list = [ti.i32, ti.i64, ti.f32, ti.f64]
-    test_dsize_list = [(4**i) * kibibyte
-                       for i in range(1, 10)]  #[4KB,16KB...256MB]
+    test_dsize_list = [
+        (4**i) * 1024  # kibibytes(KiB) = 1024
+        for i in range(1, 10)  # [4KB,16KB...256MB]
+    ]
     basic_repeat_times = 10
     evaluator = [geometric_mean]
 
     def __init__(self, arch):
-        self.arch = arch
-        self.cases_impl = []
+        self._arch = arch
+        self._cases_impl = []
         for case in self.test_cases:
             for dtype in self.test_dtype_list:
                 impl = CaseImpl(case, arch, dtype, self.test_dsize_list,
                                 self.evaluator)
-                self.cases_impl.append(impl)
+                self._cases_impl.append(impl)
 
     def run(self):
-        for case in self.cases_impl:
+        for case in self._cases_impl:
             case.run()
 
-    def get_markdown_lines(self):
+    def save_as_json(self, arch_dir='./'):
+        #folder of suite
+        suite_path = os.path.join(arch_dir, self.suite_name)
+        os.makedirs(suite_path)
+        #json files
+        self._save_suite_info_as_json(suite_path)
+        self._save_cases_info_as_json(suite_path)
+
+    def save_as_markdown(self, arch_dir='./'):
+        current_time = datatime_with_format()
+        commit_hash = ti.core.get_commit_hash()  #[:8]
+        file_name = f'{self.suite_name}.md'
+        file_path = os.path.join(arch_dir, file_name)
+        with open(file_path, 'w') as f:
+            lines = [
+                f'commit_hash: {commit_hash}\n', f'datatime: {current_time}\n'
+            ]
+            lines += self._get_markdown_lines()
+            for line in lines:
+                print(line, file=f)
+
+    def _save_suite_info_as_json(self, suite_path='./'):
+        info_dict = {
+            'cases': [func.__name__ for func in self.test_cases],
+            'dtype': [dtype2str(dtype) for dtype in self.test_dtype_list],
+            'dsize': [size for size in self.test_dsize_list],
+            'repeat': [
+                scaled_repeat_times(self._arch, size, self.basic_repeat_times)
+                for size in self.test_dsize_list
+            ],
+            'evaluator': [func.__name__ for func in self.evaluator]
+        }
+        info_path = os.path.join(suite_path, '_info.json')
+        with open(info_path, 'w') as f:
+            print(dump2json(info_dict), file=f)
+
+    def _save_cases_info_as_json(self, suite_path='./'):
+        for case in self.test_cases:  #for case [fill,saxpy,reduction]
+            results_dict = {}
+            for impl in self._cases_impl:  #find [ti.i32, ti.i64, ti.f32, ti.f64]
+                if impl._name != case.__name__:
+                    continue
+                result_name = dtype2str(impl._test_dtype)
+                results_dict[result_name] = impl.get_results_dict()
+            case_path = os.path.join(suite_path, (case.__name__ + '.json'))
+            with open(case_path, 'w') as f:
+                case_str = dump2json(results_dict)
+                print(case_str, file=f)
+
+    def _get_markdown_lines(self):
         lines = []
-        lines += md_table_header(self.suite_name, self.arch,
+        lines += md_table_header(self.suite_name, self._arch,
                                  self.test_dsize_list, self.basic_repeat_times,
                                  self.evaluator)
 
         result_header = '|kernel elapsed time(ms)' + ''.join(
             '|' for i in range(
                 len(self.test_dsize_list) + len(MemoryBound.evaluator)))
         lines += [result_header]
-        for case in self.cases_impl:
+        for case in self._cases_impl:
             lines += case.get_markdown_lines()
         lines.append('')
         return lines
 
 
 class CaseImpl:
     def __init__(self, func, arch, test_dtype, test_dsize_list, evaluator):
-        self.func = func
-        self.name = func.__name__
-        self.arch = arch
-        self.test_dtype = test_dtype
-        self.test_dsize_list = test_dsize_list
-        self.min_time_in_us = []  #test results
-        self.evaluator = evaluator
+        self._func = func
+        self._name = func.__name__
+        self._arch = arch
+        self._test_dtype = test_dtype
+        self._test_dsize_list = test_dsize_list
+        self._min_time_in_us = []  #test results
+        self._evaluator = evaluator
 
     def run(self):
-        ti.init(kernel_profiler=True, arch=self.arch)
-        print("TestCase[%s.%s.%s]" % (self.func.__name__, arch_name(
-            self.arch), dtype2str[self.test_dtype]))
-        for test_dsize in self.test_dsize_list:
+        ti.init(kernel_profiler=True, arch=self._arch)
+        print("TestCase[%s.%s.%s]" % (self._func.__name__, arch_name(
+            self._arch), dtype2str(self._test_dtype)))
+        for test_dsize in self._test_dsize_list:
             print("test_dsize = %s" % (size2str(test_dsize)))
-            self.min_time_in_us.append(
-                self.func(self.arch, self.test_dtype, test_dsize,
-                          MemoryBound.basic_repeat_times))
+            self._min_time_in_us.append(
+                self._func(self._arch, self._test_dtype, test_dsize,
+                           MemoryBound.basic_repeat_times))
             time.sleep(0.2)
         ti.reset()
 
     def get_markdown_lines(self):
-        string = '|' + self.name + '.' + dtype2str[self.test_dtype] + '|'
+        string = '|' + self._name + '.' + dtype2str(self._test_dtype) + '|'
         string += ''.join(
-            str(round(time, 4)) + '|' for time in self.min_time_in_us)
+            str(round(time, 4)) + '|' for time in self._min_time_in_us)
         string += ''.join(
-            str(round(item(self.min_time_in_us), 4)) + '|'
-            for item in self.evaluator)
+            str(round(item(self._min_time_in_us), 4)) + '|'
+            for item in self._evaluator)
         return [string]
+
+    def get_results_dict(self):
+        results_dict = {}
+        for i in range(len(self._test_dsize_list)):
+            dsize = self._test_dsize_list[i]
+            repeat = scaled_repeat_times(self._arch, dsize,
+                                         MemoryBound.basic_repeat_times)
+            elapsed_time = self._min_time_in_us[i]
+            item_name = size2str(dsize).replace('.0', '')
+            item_dict = {
+                'dsize_byte': dsize,
+                'repeat': repeat,
+                'elapsed_time_ms': elapsed_time
+            }
+            results_dict[item_name] = item_dict
+        return results_dict

benchmarks/misc/membound_cases.py

@@ -26,7 +26,7 @@ def membound_benchmark(func, num_elements, repeat):
 def fill(arch, dtype, dsize, repeat=10):
 
     repeat = scaled_repeat_times(arch, dsize, repeat)
-    num_elements = dsize // dtype_size[dtype]
+    num_elements = dsize // dtype_size(dtype)
 
     x = ti.field(dtype, shape=num_elements)
 
@@ -41,7 +41,7 @@ def fill_const(n: ti.i32):
 def saxpy(arch, dtype, dsize, repeat=10):
 
     repeat = scaled_repeat_times(arch, dsize, repeat)
-    num_elements = dsize // dtype_size[dtype] // 3  #z=x+y
+    num_elements = dsize // dtype_size(dtype) // 3  #z=x+y
 
     x = ti.field(dtype, shape=num_elements)
     y = ti.field(dtype, shape=num_elements)
@@ -61,7 +61,7 @@ def saxpy(n: ti.i32):
 def reduction(arch, dtype, dsize, repeat=10):
 
     repeat = scaled_repeat_times(arch, dsize, repeat)
-    num_elements = dsize // dtype_size[dtype]
+    num_elements = dsize // dtype_size(dtype)
 
     x = ti.field(dtype, shape=num_elements)
     y = ti.field(dtype, shape=())

benchmarks/misc/run.py

@@ -2,32 +2,54 @@
 import warnings
 
 from membound import MemoryBound
-from taichi.core import ti_core as _ti_core
-from utils import arch_name, datatime_with_format
+from utils import arch_name, datatime_with_format, dump2json
 
 import taichi as ti
 
 benchmark_suites = [MemoryBound]
-benchmark_archs = [ti.cpu, ti.cuda]
+benchmark_archs = [ti.x64, ti.cuda]
 
 
-class CommitInfo:
-    def __init__(self, pull_request_id, commit_hash):
+class BenchmarksInfo:
+    def __init__(self, pull_request_id: str, commit_hash: str):
+        """init with commit info"""
         self.pull_request_id = pull_request_id
-        self.commit_hash = commit_hash  #str
-        self.archs = []  #['x64','cuda','vulkan', ...]
+        self.commit_hash = commit_hash
         self.datetime = []  #[start, end]
+        self.archs = {}
+        # "archs": {
+        #     "x64": ["memorybound"], #arch:[suites name]
+        #     "cuda": ["memorybound"]
+        # }
+    def add_suites_info(self, arch, suites):
+        self.archs[arch_name(arch)] = suites.get_suites_name()
 
 
 class BenchmarkSuites:
     def __init__(self, arch):
-        self.suites = []
-        self.arch = arch
+        self._suites = []
+        self._arch = arch
         for suite in benchmark_suites:
-            if self.check_supported(arch, suite):
-                self.suites.append(suite(arch))
+            if self._check_supported(arch, suite):
+                self._suites.append(suite(arch))
 
-    def check_supported(self, arch, suite):
+    def run(self):
+        print(f'Arch [{arch_name(self._arch)}] Running...')
+        for suite in self._suites:
+            suite.run()
+
+    def save(self, benchmark_dir='./'):
+        #folder of archs
+        arch_dir = os.path.join(benchmark_dir, arch_name(self._arch))
+        os.makedirs(arch_dir)
+        for suite in self._suites:
+            suite.save_as_json(arch_dir)
+            suite.save_as_markdown(arch_dir)
+
+    def get_suites_name(self):
+        return [suite.suite_name for suite in self._suites]
+
+    def _check_supported(self, arch, suite):
         if arch in suite.supported_archs:
             return True
         else:
@@ -37,41 +59,35 @@ def check_supported(self, arch, suite):
                 stacklevel=2)
             return False
 
-    def run(self):
-        print(f'Arch [{arch_name(self.arch)}] Running...')
-        for suite in self.suites:
-            suite.run()
-
-    def save_to_markdown(self, arch_dir='./'):
-        current_time = datatime_with_format()
-        commit_hash = _ti_core.get_commit_hash()  #[:8]
-        for suite in self.suites:
-            file_name = f'{suite.suite_name}.md'
-            path = os.path.join(arch_dir, file_name)
-            with open(path, 'w') as f:
-                lines = [
-                    f'commit_hash: {commit_hash}\n',
-                    f'datatime: {current_time}\n'
-                ]
-                lines += suite.get_markdown_lines()
-                for line in lines:
-                    print(line, file=f)
-
 
 def main():
 
     benchmark_dir = os.path.join(os.getcwd(), 'results')
     os.makedirs(benchmark_dir)
 
+    pull_request_id = os.environ.get('PULL_REQUEST_NUMBER')
+    commit_hash = ti.core.get_commit_hash()  #[:8]
+    info = BenchmarksInfo(pull_request_id, commit_hash)
+    info.datetime.append(datatime_with_format())  #start time
+
+    print(f'pull_request_id = {pull_request_id}')
+    print(f'commit_hash = {commit_hash}')
+
     for arch in benchmark_archs:
-        #make dir
-        arch_dir = os.path.join(benchmark_dir, arch_name(arch))
-        os.makedirs(arch_dir)
         #init & run
         suites = BenchmarkSuites(arch)
         suites.run()
         #save result
-        suites.save_to_markdown(arch_dir)
+        suites.save(benchmark_dir)
+        #add benchmark info
+        info.add_suites_info(arch, suites)
+
+    info.datetime.append(datatime_with_format())  #end time
+    #save commit and benchmark info
+    info_path = os.path.join(benchmark_dir, '_info.json')
+    info_str = dump2json(info)
+    with open(info_path, 'w') as f:
+        print(info_str, file=f)
 
 
 if __name__ == '__main__':

benchmarks/misc/utils.py

@@ -1,18 +1,30 @@
 import datetime
+import json
+
+import jsbeautifier
 
 import taichi as ti
 
-kibibyte = 1024
 
-bls2str = {False: "BLS_off", True: "BLS_on"}
-dense2str = {False: "Struct_for", True: "Range_for"}
+def dtype2str(ti_dtype):
+    type_str_dict = {
+        ti.i32: "i32",
+        ti.i64: "i64",
+        ti.f32: "f32",
+        ti.f64: "f64"
+    }
+    if ti_dtype not in type_str_dict:
+        raise RuntimeError('Unsupported ti.dtype: ' + str(type(ti_dtype)))
+    else:
+        return type_str_dict[ti_dtype]
 
-dtype2str = {ti.i32: "i32", ti.i64: "i64", ti.f32: "f32", ti.f64: "f64"}
-dtype_size = {ti.i32: 4, ti.i64: 8, ti.f32: 4, ti.f64: 8}
 
-# for output string
-size_subsection = [(0.0, 'B'), (1024.0, 'KB'), (1048576.0, 'MB'),
-                   (1073741824.0, 'GB'), (float('inf'), 'INF')]  #B KB MB GB
+def dtype_size(ti_dtype):
+    dtype_size_dict = {ti.i32: 4, ti.i64: 8, ti.f32: 4, ti.f64: 8}
+    if ti_dtype not in dtype_size_dict:
+        raise RuntimeError('Unsupported ti.dtype: ' + str(type(ti_dtype)))
+    else:
+        return dtype_size_dict[ti_dtype]
 
 
 def arch_name(arch):
@@ -23,7 +35,21 @@ def datatime_with_format():
     return datetime.datetime.now().isoformat()
 
 
+def dump2json(obj):
+    if type(obj) is dict:
+        obj2dict = obj
+    else:
+        obj2dict = obj.__dict__
+    options = jsbeautifier.default_options()
+    options.indent_size = 4
+    return jsbeautifier.beautify(json.dumps(obj2dict), options)
+
+
 def size2str(size_in_byte):
+    # for output string
+    size_subsection = [(0.0, 'B'), (1024.0, 'KB'), (1048576.0, 'MB'),
+                       (1073741824.0, 'GB'),
+                       (float('inf'), 'INF')]  #B KB MB GB
     for dsize, units in reversed(size_subsection):
         if size_in_byte >= dsize:
             return str(round(size_in_byte / dsize, 4)) + units