feat: implement batching strategies

src/bentoml/_internal/marshal/dispatcher.py

@@ -124,6 +124,8 @@ def __init__(self, options: dict[str, t.Any]):
         self.o_stat: collections.deque[tuple[int, float, float]] = collections.deque(
             maxlen=self.n_kept_sample
         )  # to store outbound stat data
+        self.o_a = min(2, max_latency * 2.0 / 30)
+        self.o_b = min(1, max_latency * 1.0 / 30)
 
         self.wait = 0  # the avg wait time before outbound called
 

src/bentoml/_internal/runner/runner.py

@@ -268,7 +268,6 @@ def __init__(
                     config["batching"]["strategy"]
                 ](config["batching"]["strategy_options"])
             except Exception as e:
-                raise e
                 raise BentoMLConfigException(
                     f"Initializing strategy '{config['batching']['strategy']}' with configured options ({pprint(config['batching']['strategy_options'])}) failed."
                 ) from e

src/bentoml/_internal/server/runner_app.py

@@ -61,6 +61,7 @@ def __init__(
 
             self.dispatchers[method.name] = Dispatcher(
                 max_latency_in_ms=method.max_latency_ms,
+                batching_strategy=method.batching_strategy,
                 max_batch_size=max_batch_size,
                 optimizer=method.optimizer,
                 strategy=method.batching_strategy,