From 416323257c1dd07198e12a55a32ca2aef27dcf6e Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 24 Jul 2024 18:54:11 +0300 Subject: [PATCH] =?UTF-8?q?=E2=9C=85=20`click`=20CLI=20interface=20is=20pr?= =?UTF-8?q?epared=20and=20tested?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/__init__.py | 4 -- src/guidellm/main.py | 20 ++++-- tests/unit/cli/__init__.py | 0 tests/unit/cli/conftest.py | 39 +++++++++++ tests/unit/cli/test_application_entrypoint.py | 64 +++++++++++++++++++ tests/unit/cli/test_main_validation.py | 37 +++++++++++ tests/unit/test_logger.py | 3 +- 7 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 tests/unit/cli/__init__.py create mode 100644 tests/unit/cli/conftest.py create mode 100644 tests/unit/cli/test_application_entrypoint.py create mode 100644 tests/unit/cli/test_main_validation.py diff --git a/src/guidellm/__init__.py b/src/guidellm/__init__.py index 7be55c3..c5b6b57 100644 --- a/src/guidellm/__init__.py +++ b/src/guidellm/__init__.py @@ -2,7 +2,3 @@ Guidellm is a package that provides an easy and intuitive interface for evaluating and benchmarking large language models (LLMs). """ - -from .logger import configure_logger, logger - -__all__ = ["logger", "configure_logger"] diff --git a/src/guidellm/main.py b/src/guidellm/main.py index bf7601c..e48ab71 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -23,15 +23,20 @@ default="localhost:8000/completions", help="Target for benchmarking", ) -@click.option("--host", type=str, help="Host for benchmarking") -@click.option("--port", type=str, help="Port for benchmarking") -@click.option("--path", type=str, help="Path for benchmarking") +@click.option("--host", type=str, default=None, help="Host for benchmarking") +@click.option("--port", type=str, default=None, help="Port for benchmarking") +@click.option("--path", type=str, default=None, help="Path for benchmarking") @click.option( - "--backend", type=str, default="openai_server", help="Backend type for benchmarking" + "--backend", + type=click.Choice(["test", "openai_server"]), + default="openai_server", + help="Backend type for benchmarking", ) @click.option("--model", type=str, default=None, help="Model to use for benchmarking") @click.option("--task", type=str, default=None, help="Task to use for benchmarking") -@click.option("--data", type=str, help="Data file or alias for benchmarking") +@click.option( + "--data", type=str, default=None, help="Data file or alias for benchmarking" +) @click.option( "--data-type", type=click.Choice(["emulated", "file", "transformers"]), @@ -44,7 +49,7 @@ @click.option( "--rate-type", type=click.Choice(["sweep", "synchronous", "constant", "poisson"]), - default="sweep", + default="synchronous", help="Type of rate generation for benchmarking", ) @click.option( @@ -57,7 +62,7 @@ @click.option( "--num-seconds", type=int, - default="120", + default=120, help="Number of seconds to result each request rate at", ) @click.option( @@ -82,6 +87,7 @@ def main( num_seconds, num_requests, ): + # Create backend Backend.create( backend_type=backend, diff --git a/tests/unit/cli/__init__.py b/tests/unit/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py new file mode 100644 index 0000000..faff2c0 --- /dev/null +++ b/tests/unit/cli/conftest.py @@ -0,0 +1,39 @@ +from typing import Any, Dict +from unittest.mock import MagicMock + +import pytest +from click.testing import CliRunner + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +@pytest.fixture +def patch_main(mocker) -> MagicMock: + return mocker.patch("guidellm.main.main.callback") + + +@pytest.fixture +def default_main_kwargs() -> Dict[str, Any]: + """ + All the defaults come from the `guidellm.main` function. + """ + + return { + "target": "localhost:8000/completions", + "host": None, + "port": None, + "path": None, + "backend": "openai_server", + "model": None, + "task": None, + "data": None, + "data_type": "transformers", + "tokenizer": None, + "rate_type": "synchronous", + "rate": (1.0,), + "num_seconds": 120, + "num_requests": None, + } diff --git a/tests/unit/cli/test_application_entrypoint.py b/tests/unit/cli/test_application_entrypoint.py new file mode 100644 index 0000000..6aa1802 --- /dev/null +++ b/tests/unit/cli/test_application_entrypoint.py @@ -0,0 +1,64 @@ +from typing import List +from unittest.mock import MagicMock + +import pytest +from click.testing import CliRunner + +from guidellm.main import main + + +def test_main_defaults_from_cli( + patch_main: MagicMock, cli_runner: CliRunner, default_main_kwargs +): + cli_runner.invoke(main) + + assert patch_main.call_count == 1 + assert patch_main.call_args.kwargs == default_main_kwargs + + +def test_main_cli_overrided( + patch_main: MagicMock, cli_runner: CliRunner, default_main_kwargs +): + cli_runner.invoke( + main, + ["--target", "localhost:9000", "--backend", "test", "--rate-type", "sweep"], + ) + default_main_kwargs.update( + {"target": "localhost:9000", "backend": "test", "rate_type": "sweep"} + ) + + assert patch_main.call_count == 1 + assert patch_main.call_args.kwargs == default_main_kwargs + + +@pytest.mark.parametrize( + "args,expected_stdout", + [ + ( + ["--backend", "invalid", "--rate-type", "sweep"], + ( + b"Usage: main [OPTIONS]\nTry 'main --help' for help.\n\n" + b"Error: Invalid value for '--backend': " + b"'invalid' is not one of 'test', 'openai_server'.\n" + ), + ), + ( + ["--num-requests", "str instead of int"], + ( + b"Usage: main [OPTIONS]\nTry 'main --help' for help.\n\n" + b"Error: Invalid value for '--num-requests': " + b"'str instead of int' is not a valid integer.\n" + ), + ), + ], +) +def test_main_cli_validation_error( + patch_main: MagicMock, + cli_runner: CliRunner, + args: List[str], + expected_stdout: bytes, +): + result = cli_runner.invoke(main, args) + + assert patch_main.call_count == 0 + assert result.stdout_bytes == expected_stdout diff --git a/tests/unit/cli/test_main_validation.py b/tests/unit/cli/test_main_validation.py new file mode 100644 index 0000000..b529048 --- /dev/null +++ b/tests/unit/cli/test_main_validation.py @@ -0,0 +1,37 @@ +import pytest + +from guidellm.main import main + + +def test_task_without_data(mocker, default_main_kwargs): + patch = mocker.patch("guidellm.backend.Backend.create") + default_main_kwargs.update({"task": "can't be used without data"}) + with pytest.raises(NotImplementedError): + getattr(main, "callback")(**default_main_kwargs) + + assert patch.call_count == 1 + + +def test_invalid_data_type(mocker, default_main_kwargs): + patch = mocker.patch("guidellm.backend.Backend.create") + default_main_kwargs.update({"data_type": "invalid"}) + + with pytest.raises(ValueError): + getattr(main, "callback")(**default_main_kwargs) + + assert patch.call_count == 1 + + +def test_invalid_rate_type(mocker, default_main_kwargs): + patch = mocker.patch("guidellm.backend.Backend.create") + file_request_generator_initialization_patch = mocker.patch( + "guidellm.request.file.FileRequestGenerator.__init__", + return_value=None, + ) + default_main_kwargs.update({"rate_type": "invalid", "data_type": "file"}) + + with pytest.raises(ValueError): + getattr(main, "callback")(**default_main_kwargs) + + assert patch.call_count == 1 + assert file_request_generator_initialization_patch.call_count == 1 diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index fb3240a..87141a1 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -1,7 +1,8 @@ import pytest +from loguru import logger from config import LoggingSettings -from guidellm import configure_logger, logger +from guidellm.logger import configure_logger @pytest.fixture(autouse=True)