✅ click CLI interface is prepared and tested

src/guidellm/__init__.py

@@ -2,7 +2,3 @@
 Guidellm is a package that provides an easy and intuitive interface for
 evaluating and benchmarking large language models (LLMs).
 """
-
-from .logger import configure_logger, logger
-
-__all__ = ["logger", "configure_logger"]

src/guidellm/main.py

@@ -23,15 +23,20 @@
     default="localhost:8000/completions",
     help="Target for benchmarking",
 )
-@click.option("--host", type=str, help="Host for benchmarking")
-@click.option("--port", type=str, help="Port for benchmarking")
-@click.option("--path", type=str, help="Path for benchmarking")
+@click.option("--host", type=str, default=None, help="Host for benchmarking")
+@click.option("--port", type=str, default=None, help="Port for benchmarking")
+@click.option("--path", type=str, default=None, help="Path for benchmarking")
 @click.option(
-    "--backend", type=str, default="openai_server", help="Backend type for benchmarking"
+    "--backend",
+    type=click.Choice(["test", "openai_server"]),
+    default="openai_server",
+    help="Backend type for benchmarking",
 )
 @click.option("--model", type=str, default=None, help="Model to use for benchmarking")
 @click.option("--task", type=str, default=None, help="Task to use for benchmarking")
-@click.option("--data", type=str, help="Data file or alias for benchmarking")
+@click.option(
+    "--data", type=str, default=None, help="Data file or alias for benchmarking"
+)
 @click.option(
     "--data-type",
     type=click.Choice(["emulated", "file", "transformers"]),
@@ -44,7 +49,7 @@
 @click.option(
     "--rate-type",
     type=click.Choice(["sweep", "synchronous", "constant", "poisson"]),
-    default="sweep",
+    default="synchronous",
     help="Type of rate generation for benchmarking",
 )
 @click.option(
@@ -57,7 +62,7 @@
 @click.option(
     "--num-seconds",
     type=int,
-    default="120",
+    default=120,
     help="Number of seconds to result each request rate at",
 )
 @click.option(
@@ -82,6 +87,7 @@ def main(
     num_seconds,
     num_requests,
 ):
+
     # Create backend
     Backend.create(
         backend_type=backend,

tests/unit/cli/conftest.py

@@ -0,0 +1,39 @@
+from typing import Any, Dict
+from unittest.mock import MagicMock
+
+import pytest
+from click.testing import CliRunner
+
+
+@pytest.fixture
+def cli_runner():
+    return CliRunner()
+
+
+@pytest.fixture
+def patch_main(mocker) -> MagicMock:
+    return mocker.patch("guidellm.main.main.callback")
+
+
+@pytest.fixture
+def default_main_kwargs() -> Dict[str, Any]:
+    """
+    All the defaults come from the `guidellm.main` function.
+    """
+
+    return {
+        "target": "localhost:8000/completions",
+        "host": None,
+        "port": None,
+        "path": None,
+        "backend": "openai_server",
+        "model": None,
+        "task": None,
+        "data": None,
+        "data_type": "transformers",
+        "tokenizer": None,
+        "rate_type": "synchronous",
+        "rate": (1.0,),
+        "num_seconds": 120,
+        "num_requests": None,
+    }

tests/unit/cli/test_application_entrypoint.py

@@ -0,0 +1,64 @@
+from typing import List
+from unittest.mock import MagicMock
+
+import pytest
+from click.testing import CliRunner
+
+from guidellm.main import main
+
+
+def test_main_defaults_from_cli(
+    patch_main: MagicMock, cli_runner: CliRunner, default_main_kwargs
+):
+    cli_runner.invoke(main)
+
+    assert patch_main.call_count == 1
+    assert patch_main.call_args.kwargs == default_main_kwargs
+
+
+def test_main_cli_overrided(
+    patch_main: MagicMock, cli_runner: CliRunner, default_main_kwargs
+):
+    cli_runner.invoke(
+        main,
+        ["--target", "localhost:9000", "--backend", "test", "--rate-type", "sweep"],
+    )
+    default_main_kwargs.update(
+        {"target": "localhost:9000", "backend": "test", "rate_type": "sweep"}
+    )
+
+    assert patch_main.call_count == 1
+    assert patch_main.call_args.kwargs == default_main_kwargs
+
+
+@pytest.mark.parametrize(
+    "args,expected_stdout",
+    [
+        (
+            ["--backend", "invalid", "--rate-type", "sweep"],
+            (
+                b"Usage: main [OPTIONS]\nTry 'main --help' for help.\n\n"
+                b"Error: Invalid value for '--backend': "
+                b"'invalid' is not one of 'test', 'openai_server'.\n"
+            ),
+        ),
+        (
+            ["--num-requests", "str instead of int"],
+            (
+                b"Usage: main [OPTIONS]\nTry 'main --help' for help.\n\n"
+                b"Error: Invalid value for '--num-requests': "
+                b"'str instead of int' is not a valid integer.\n"
+            ),
+        ),
+    ],
+)
+def test_main_cli_validation_error(
+    patch_main: MagicMock,
+    cli_runner: CliRunner,
+    args: List[str],
+    expected_stdout: bytes,
+):
+    result = cli_runner.invoke(main, args)
+
+    assert patch_main.call_count == 0
+    assert result.stdout_bytes == expected_stdout

tests/unit/cli/test_main_validation.py

@@ -0,0 +1,37 @@
+import pytest
+
+from guidellm.main import main
+
+
+def test_task_without_data(mocker, default_main_kwargs):
+    patch = mocker.patch("guidellm.backend.Backend.create")
+    default_main_kwargs.update({"task": "can't be used without data"})
+    with pytest.raises(NotImplementedError):
+        getattr(main, "callback")(**default_main_kwargs)
+
+    assert patch.call_count == 1
+
+
+def test_invalid_data_type(mocker, default_main_kwargs):
+    patch = mocker.patch("guidellm.backend.Backend.create")
+    default_main_kwargs.update({"data_type": "invalid"})
+
+    with pytest.raises(ValueError):
+        getattr(main, "callback")(**default_main_kwargs)
+
+    assert patch.call_count == 1
+
+
+def test_invalid_rate_type(mocker, default_main_kwargs):
+    patch = mocker.patch("guidellm.backend.Backend.create")
+    file_request_generator_initialization_patch = mocker.patch(
+        "guidellm.request.file.FileRequestGenerator.__init__",
+        return_value=None,
+    )
+    default_main_kwargs.update({"rate_type": "invalid", "data_type": "file"})
+
+    with pytest.raises(ValueError):
+        getattr(main, "callback")(**default_main_kwargs)
+
+    assert patch.call_count == 1
+    assert file_request_generator_initialization_patch.call_count == 1

tests/unit/test_logger.py

@@ -1,7 +1,8 @@
 import pytest
+from loguru import logger
 
 from config import LoggingSettings
-from guidellm import configure_logger, logger
+from guidellm.logger import configure_logger
 
 
 @pytest.fixture(autouse=True)