From ae42ac9d4927b8dbca410428b7f56a5752ced957 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sun, 3 Dec 2023 21:29:22 +0400 Subject: [PATCH 01/13] refactor repeater --- tests/plugins/repeater/_utils.py | 19 +++- tests/plugins/repeater/test_repeater.py | 136 ++++++++++++++---------- vedro/plugins/repeater/_repeater.py | 35 +++--- 3 files changed, 115 insertions(+), 75 deletions(-) diff --git a/tests/plugins/repeater/_utils.py b/tests/plugins/repeater/_utils.py index 51702a0b..f0ded49e 100644 --- a/tests/plugins/repeater/_utils.py +++ b/tests/plugins/repeater/_utils.py @@ -2,6 +2,7 @@ from argparse import ArgumentParser, Namespace from pathlib import Path from time import monotonic_ns +from typing import Optional from unittest.mock import AsyncMock, Mock import pytest @@ -22,6 +23,7 @@ ArgParseEvent, ConfigLoadedEvent, ScenarioFailedEvent, + ScenarioPassedEvent, StartupEvent, ) from vedro.plugins.repeater import Repeater, RepeaterPlugin @@ -63,8 +65,10 @@ class _Scenario(Scenario): return VirtualScenario(_Scenario, steps=[]) -def make_scenario_result() -> ScenarioResult: - return ScenarioResult(make_vscenario()) +def make_scenario_result(scenario: Optional[VirtualScenario] = None) -> ScenarioResult: + if scenario is None: + scenario = make_vscenario() + return ScenarioResult(scenario) def make_config() -> ConfigType: @@ -92,8 +96,15 @@ async def fire_startup_event(dispatcher: Dispatcher, scheduler: Scheduler) -> No await dispatcher.fire(startup_event) -async def fire_failed_event(dispatcher: Dispatcher) -> ScenarioFailedEvent: +async def fire_passed_event(dispatcher: Dispatcher) -> ScenarioResult: + scenario_result = make_scenario_result().mark_passed() + scenario_passed_event = ScenarioPassedEvent(scenario_result) + await dispatcher.fire(scenario_passed_event) + return scenario_result + + +async def fire_failed_event(dispatcher: Dispatcher) -> ScenarioResult: scenario_result = make_scenario_result().mark_failed() scenario_failed_event = ScenarioFailedEvent(scenario_result) await dispatcher.fire(scenario_failed_event) - return scenario_failed_event + return scenario_result diff --git a/tests/plugins/repeater/test_repeater.py b/tests/plugins/repeater/test_repeater.py index eac79e9a..12d617b3 100644 --- a/tests/plugins/repeater/test_repeater.py +++ b/tests/plugins/repeater/test_repeater.py @@ -1,10 +1,11 @@ +from typing import Callable from unittest.mock import AsyncMock, Mock, call import pytest from baby_steps import given, then, when from pytest import raises -from vedro.core import Dispatcher, Report +from vedro.core import Dispatcher, Event, Report, ScenarioResult from vedro.events import ( CleanupEvent, ScenarioFailedEvent, @@ -16,6 +17,7 @@ dispatcher, fire_arg_parsed_event, fire_failed_event, + fire_passed_event, fire_startup_event, make_scenario_result, repeater, @@ -26,71 +28,77 @@ __all__ = ("dispatcher", "repeater", "scheduler_", "sleep_") # fixtures +@pytest.mark.parametrize("repeats", [2, 3]) +@pytest.mark.parametrize("get_event", [ + lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), + lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), +]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=0) - - with then: - assert exc_info.type is ValueError - assert str(exc_info.value) == "--repeats must be >= 1" - - -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_delay_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=-0.001) - - with then: - assert exc_info.type is ValueError - assert str(exc_info.value) == "--repeats-delay must be >= 0.0" +async def test_repeat(repeats: int, get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): + with given: + await fire_arg_parsed_event(dispatcher, repeats=repeats) + await fire_startup_event(dispatcher, scheduler_) + scenario_result = make_scenario_result() + scenario_event = get_event(scenario_result) -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_delay_without_repeats_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=1, repeats_delay=0.1) + with when: + await dispatcher.fire(scenario_event) with then: - assert exc_info.type is ValueError - assert str(exc_info.value) == "--repeats-delay must be used with --repeats > 1" + assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] + assert sleep_.mock_calls == [] -@pytest.mark.parametrize("repeats", [1, 2, 3]) +@pytest.mark.parametrize("get_event", [ + lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), + lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), +]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_passed(repeats: int, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_single_repeat_fired(get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: - await fire_arg_parsed_event(dispatcher, repeats=repeats) + await fire_arg_parsed_event(dispatcher, repeats=2) await fire_startup_event(dispatcher, scheduler_) - scenario_result = make_scenario_result().mark_passed() - scenario_passed_event = ScenarioPassedEvent(scenario_result) + scenario_result1 = await fire_passed_event(dispatcher) + scenario_result2 = make_scenario_result(scenario_result1.scenario) + + scenario_event = get_event(scenario_result2) + scheduler_.reset_mock() with when: - await dispatcher.fire(scenario_passed_event) + await dispatcher.fire(scenario_event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] * (repeats - 1) + assert scheduler_.mock_calls == [] assert sleep_.mock_calls == [] -@pytest.mark.parametrize("repeats", [1, 2, 3]) +@pytest.mark.parametrize("repeats", [3, 4]) +@pytest.mark.parametrize("get_event", [ + lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), + lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), +]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_failed(repeats: int, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_multiple_repeat_fired(repeats: int, get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=repeats) await fire_startup_event(dispatcher, scheduler_) - scenario_result = make_scenario_result().mark_failed() - scenario_failed_event = ScenarioFailedEvent(scenario_result) + scenario_result1 = await fire_passed_event(dispatcher) + scenario_result2 = make_scenario_result(scenario_result1.scenario) + + scenario_event = get_event(scenario_result2) + scheduler_.reset_mock() with when: - await dispatcher.fire(scenario_failed_event) + await dispatcher.fire(scenario_event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] * (repeats - 1) + assert scheduler_.mock_calls == [call.schedule(scenario_result2.scenario)] assert sleep_.mock_calls == [] @@ -113,23 +121,6 @@ async def test_dont_repeat_skipped(repeats: int, *, assert sleep_.mock_calls == [] -@pytest.mark.usefixtures(repeater.__name__) -async def test_dont_repeat_repeated(dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): - with given: - await fire_arg_parsed_event(dispatcher, repeats=2) - await fire_startup_event(dispatcher, scheduler_) - - scenario_failed_event = await fire_failed_event(dispatcher) - scheduler_.reset_mock() - - with when: - await dispatcher.fire(scenario_failed_event) - - with then: - assert scheduler_.mock_calls == [] - assert sleep_.mock_calls == [] - - @pytest.mark.parametrize(("repeats", "repeats_delay"), [ (2, 0.1), (3, 1.0) @@ -148,8 +139,8 @@ async def test_repeat_with_delay(repeats: int, repeats_delay: float, *, await dispatcher.fire(scenario_passed_event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] * (repeats - 1) - assert sleep_.mock_calls == [call(repeats_delay)] * (repeats - 1) + assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] + assert sleep_.mock_calls == [call(repeats_delay)] @pytest.mark.parametrize("repeats", [2, 3]) @@ -210,3 +201,32 @@ async def test_dont_add_summary(dispatcher: Dispatcher, scheduler_: Mock): with then: assert report.summary == [] + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeat_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=0) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--repeats must be >= 1" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeat_delay_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=-0.001) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--repeats-delay must be >= 0.0" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeat_delay_without_repeats_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=1, repeats_delay=0.1) + + with then: + assert exc_info.type is ValueError diff --git a/vedro/plugins/repeater/_repeater.py b/vedro/plugins/repeater/_repeater.py index 4a769be6..fb42bef0 100644 --- a/vedro/plugins/repeater/_repeater.py +++ b/vedro/plugins/repeater/_repeater.py @@ -30,6 +30,7 @@ def __init__(self, config: Type["Repeater"], *, sleep: SleepType = asyncio.sleep self._global_config: Union[ConfigType, None] = None self._scheduler: Union[ScenarioScheduler, None] = None self._repeat_scenario_id: Union[str, None] = None + self._repeat_count: int = 0 def subscribe(self, dispatcher: Dispatcher) -> None: dispatcher.listen(ConfigLoadedEvent, self.on_config_loaded) \ @@ -77,24 +78,32 @@ async def on_scenario_end(self, event: Union[ScenarioPassedEvent, ScenarioFailedEvent]) -> None: if self._repeats <= 1: return - assert isinstance(self._scheduler, RepeaterScenarioScheduler) # for type checking - if self._repeat_scenario_id == event.scenario_result.scenario.unique_id: - return - - self._repeat_scenario_id = event.scenario_result.scenario.unique_id - for _ in range(self._repeats - 1): - if self._repeats_delay > 0.0: - await self._sleep(self._repeats_delay) + scenario_id = event.scenario_result.scenario.unique_id + if self._repeat_scenario_id != scenario_id: + self._repeat_scenario_id = scenario_id + self._repeat_count = 1 self._scheduler.schedule(event.scenario_result.scenario) + else: + self._repeat_count += 1 + if self._repeat_count < self._repeats: + self._scheduler.schedule(event.scenario_result.scenario) + + if self._repeats_delay > 0.0 and self._repeat_count < self._repeats: + await self._sleep(self._repeats_delay) def on_cleanup(self, event: CleanupEvent) -> None: - if self._repeats > 1: - message = f"repeated x{self._repeats}" - if self._repeats_delay > 0.0: - message += f" with delay {self._repeats_delay!r}s" - event.report.add_summary(message) + if self._repeats <= 1: + return + message = self._get_summary_message() + event.report.add_summary(message) + + def _get_summary_message(self) -> str: + message = f"repeated x{self._repeats}" + if self._repeats_delay > 0.0: + message += f" with delay {self._repeats_delay!r}s" + return message class Repeater(PluginConfig): From ef68c6c5f3476df60e1726491a193664354bdded Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sun, 3 Dec 2023 21:36:59 +0400 Subject: [PATCH 02/13] add tests --- tests/plugins/repeater/test_repeater.py | 63 +++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/tests/plugins/repeater/test_repeater.py b/tests/plugins/repeater/test_repeater.py index 12d617b3..f0ab4470 100644 --- a/tests/plugins/repeater/test_repeater.py +++ b/tests/plugins/repeater/test_repeater.py @@ -56,7 +56,7 @@ async def test_repeat(repeats: int, get_event: Callable[[ScenarioResult], Event] lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), ]) @pytest.mark.usefixtures(repeater.__name__) -async def test_single_repeat_fired(get_event: Callable[[ScenarioResult], Event], *, +async def test_repeat2_fired_twice(get_event: Callable[[ScenarioResult], Event], *, dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=2) @@ -82,8 +82,8 @@ async def test_single_repeat_fired(get_event: Callable[[ScenarioResult], Event], lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), ]) @pytest.mark.usefixtures(repeater.__name__) -async def test_multiple_repeat_fired(repeats: int, get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_repeat3_fired_twice(repeats: int, get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=repeats) await fire_startup_event(dispatcher, scheduler_) @@ -143,6 +143,63 @@ async def test_repeat_with_delay(repeats: int, repeats_delay: float, *, assert sleep_.mock_calls == [call(repeats_delay)] +@pytest.mark.parametrize("get_event", [ + lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), + lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), +]) +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeat2_with_delay_fired_twice(get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, + scheduler_: Mock, sleep_: AsyncMock): + with given: + await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=0.1) + await fire_startup_event(dispatcher, scheduler_) + + scenario_result1 = await fire_failed_event(dispatcher) + scenario_result2 = make_scenario_result(scenario_result1.scenario) + + scenario_event = get_event(scenario_result2) + scheduler_.reset_mock() + sleep_.reset_mock() + + with when: + await dispatcher.fire(scenario_event) + + with then: + assert scheduler_.mock_calls == [] + assert sleep_.mock_calls == [] + + +@pytest.mark.parametrize("repeats", [3, 4]) +@pytest.mark.parametrize("get_event", [ + lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), + lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), +]) +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeat3_with_delay_fired_twice(repeats: int, + get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, + scheduler_: Mock, sleep_: AsyncMock): + with given: + repeats_delay = 0.1 + await fire_arg_parsed_event(dispatcher, repeats=repeats, repeats_delay=repeats_delay) + await fire_startup_event(dispatcher, scheduler_) + + scenario_result1 = await fire_failed_event(dispatcher) + scenario_result2 = make_scenario_result(scenario_result1.scenario) + + scenario_event = get_event(scenario_result2) + scheduler_.reset_mock() + sleep_.reset_mock() + + with when: + await dispatcher.fire(scenario_event) + + with then: + assert scheduler_.mock_calls == [call.schedule(scenario_result2.scenario)] + assert sleep_.mock_calls == [call(repeats_delay)] + + @pytest.mark.parametrize("repeats", [2, 3]) @pytest.mark.usefixtures(repeater.__name__) async def test_add_summary(repeats: int, *, dispatcher: Dispatcher, scheduler_: Mock): From d5261ff5ff18fb1d3b9a3854645afb69d0f04a8f Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sun, 3 Dec 2023 22:49:55 +0400 Subject: [PATCH 03/13] add error handling --- .../scenario_result/test_scenario_result.py | 82 ++++++++++++++++++- tests/core/test_step_result.py | 55 +++++++++++++ vedro/core/_step_result.py | 9 +- .../core/scenario_result/_scenario_result.py | 12 ++- 4 files changed, 155 insertions(+), 3 deletions(-) diff --git a/tests/core/scenario_result/test_scenario_result.py b/tests/core/scenario_result/test_scenario_result.py index 021543c6..394806b6 100644 --- a/tests/core/scenario_result/test_scenario_result.py +++ b/tests/core/scenario_result/test_scenario_result.py @@ -1,11 +1,12 @@ import os from pathlib import Path from types import MethodType -from typing import Type +from typing import Callable, Type from unittest.mock import Mock import pytest from baby_steps import given, then, when +from pytest import raises from vedro import Scenario from vedro.core import ( @@ -35,6 +36,9 @@ def virtual_scenario(scenario_: Type[scenario_]): return virtual_scenario +ChangeStatusType = Callable[[ScenarioResult], ScenarioResult] + + def test_scenario_result(): with when: subject = "" @@ -72,6 +76,27 @@ def test_scenario_result_mark_passed(*, virtual_scenario: VirtualScenario): assert scenario_result.is_passed() is True +@pytest.mark.parametrize("change_status", [ + lambda scn_result: scn_result.mark_passed(), + lambda scn_result: scn_result.mark_failed(), + lambda scn_result: scn_result.mark_skipped(), +]) +def test_marked_scenario_result_mark_passed(change_status: ChangeStatusType, *, + virtual_scenario: VirtualScenario): + with given: + scenario_result = ScenarioResult(virtual_scenario) + change_status(scenario_result) + + with when, raises(BaseException) as exc: + scenario_result.mark_passed() + + with then: + assert exc.type is RuntimeError + assert str(exc.value) == ( + "Cannot mark scenario as passed because its status has already been set" + ) + + def test_scenario_result_mark_failed(*, virtual_scenario: VirtualScenario): with given: scenario_result = ScenarioResult(virtual_scenario) @@ -84,6 +109,27 @@ def test_scenario_result_mark_failed(*, virtual_scenario: VirtualScenario): assert scenario_result.is_failed() is True +@pytest.mark.parametrize("change_status", [ + lambda scn_result: scn_result.mark_passed(), + lambda scn_result: scn_result.mark_failed(), + lambda scn_result: scn_result.mark_skipped(), +]) +def test_marked_scenario_result_mark_failed(change_status: ChangeStatusType, *, + virtual_scenario: VirtualScenario): + with given: + scenario_result = ScenarioResult(virtual_scenario) + change_status(scenario_result) + + with when, raises(BaseException) as exc: + scenario_result.mark_failed() + + with then: + assert exc.type is RuntimeError + assert str(exc.value) == ( + "Cannot mark scenario as failed because its status has already been set" + ) + + def test_scenario_result_mark_skipped(*, virtual_scenario: VirtualScenario): with given: scenario_result = ScenarioResult(virtual_scenario) @@ -96,6 +142,27 @@ def test_scenario_result_mark_skipped(*, virtual_scenario: VirtualScenario): assert scenario_result.is_skipped() is True +@pytest.mark.parametrize("change_status", [ + lambda scn_result: scn_result.mark_passed(), + lambda scn_result: scn_result.mark_failed(), + lambda scn_result: scn_result.mark_skipped(), +]) +def test_marked_scenario_result_mark_skipped(change_status: ChangeStatusType, *, + virtual_scenario: VirtualScenario): + with given: + scenario_result = ScenarioResult(virtual_scenario) + change_status(scenario_result) + + with when, raises(BaseException) as exc: + scenario_result.mark_skipped() + + with then: + assert exc.type is RuntimeError + assert str(exc.value) == ( + "Cannot mark scenario as skipped because its status has already been set" + ) + + def test_scenario_result_set_started_at(*, virtual_scenario: VirtualScenario): with given: scenario_result = ScenarioResult(virtual_scenario) @@ -237,6 +304,19 @@ def test_scenario_result_attach_artifact(*, virtual_scenario: VirtualScenario): assert res is None +def test_scenario_result_attach_incorrect_artifact(*, virtual_scenario: VirtualScenario): + with given: + scenario_result = ScenarioResult(virtual_scenario) + artifact = {} + + with when, raises(BaseException) as exc: + scenario_result.attach(artifact) + + with then: + assert exc.type is TypeError + assert str(exc.value) == "artifact must be an instance of Artifact" + + def test_scenario_result_get_artifacts(*, virtual_scenario: VirtualScenario): with given: scenario_result = ScenarioResult(virtual_scenario) diff --git a/tests/core/test_step_result.py b/tests/core/test_step_result.py index 493f861b..93c152ef 100644 --- a/tests/core/test_step_result.py +++ b/tests/core/test_step_result.py @@ -1,8 +1,10 @@ from types import MethodType +from typing import Callable from unittest.mock import Mock import pytest from baby_steps import given, then, when +from pytest import raises from vedro.core import ExcInfo, MemoryArtifact, StepResult, StepStatus, VirtualStep @@ -49,6 +51,26 @@ def test_step_result_mark_passed(*, virtual_step: VirtualStep): assert step_result.is_passed() is True +@pytest.mark.parametrize("change_status", [ + lambda step_result: step_result.mark_passed(), + lambda step_result: step_result.mark_failed(), +]) +def test_marked_step_result_mark_passed(change_status: Callable[[StepResult], StepResult], *, + virtual_step: VirtualStep): + with given: + step_result = StepResult(virtual_step) + change_status(step_result) + + with when, raises(BaseException) as exc: + step_result.mark_passed() + + with then: + assert exc.type is RuntimeError + assert str(exc.value) == ( + "Cannot mark step as passed because its status has already been set" + ) + + def test_step_result_mark_failed(*, virtual_step: VirtualStep): with given: step_result = StepResult(virtual_step) @@ -61,6 +83,26 @@ def test_step_result_mark_failed(*, virtual_step: VirtualStep): assert step_result.is_failed() is True +@pytest.mark.parametrize("change_status", [ + lambda step_result: step_result.mark_passed(), + lambda step_result: step_result.mark_failed(), +]) +def test_marked_step_result_mark_failed(change_status: Callable[[StepResult], StepResult], *, + virtual_step: VirtualStep): + with given: + step_result = StepResult(virtual_step) + change_status(step_result) + + with when, raises(BaseException) as exc: + step_result.mark_failed() + + with then: + assert exc.type is RuntimeError + assert str(exc.value) == ( + "Cannot mark step as failed because its status has already been set" + ) + + def test_step_result_set_started_at(*, virtual_step: VirtualStep): with given: step_result = StepResult(virtual_step) @@ -155,6 +197,19 @@ def test_step_result_attach_artifact(*, virtual_step: VirtualStep): assert res is None +def test_step_result_attach_incorrect_artifact(*, virtual_step: VirtualStep): + with given: + step_result = StepResult(virtual_step) + artifact = {} + + with when, raises(BaseException) as exc: + step_result.attach(artifact) + + with then: + assert exc.type is TypeError + assert str(exc.value) == "artifact must be an instance of Artifact" + + def test_step_result_get_artifacts(*, virtual_step: VirtualStep): with given: step_result = StepResult(virtual_step) diff --git a/vedro/core/_step_result.py b/vedro/core/_step_result.py index 226c9740..8d362836 100644 --- a/vedro/core/_step_result.py +++ b/vedro/core/_step_result.py @@ -43,10 +43,16 @@ def is_failed(self) -> bool: return self._status == StepStatus.FAILED def mark_failed(self) -> "StepResult": + if self._status != StepStatus.PENDING: + raise RuntimeError( + "Cannot mark step as failed because its status has already been set") self._status = StepStatus.FAILED return self def mark_passed(self) -> "StepResult": + if self._status != StepStatus.PENDING: + raise RuntimeError( + "Cannot mark step as passed because its status has already been set") self._status = StepStatus.PASSED return self @@ -81,7 +87,8 @@ def set_exc_info(self, exc_info: ExcInfo) -> "StepResult": return self def attach(self, artifact: Artifact) -> None: - assert isinstance(artifact, Artifact) + if not isinstance(artifact, Artifact): + raise TypeError("artifact must be an instance of Artifact") self._artifacts.append(artifact) @property diff --git a/vedro/core/scenario_result/_scenario_result.py b/vedro/core/scenario_result/_scenario_result.py index c88fa211..3475a09e 100644 --- a/vedro/core/scenario_result/_scenario_result.py +++ b/vedro/core/scenario_result/_scenario_result.py @@ -31,6 +31,9 @@ def status(self) -> ScenarioStatus: return self._status def mark_passed(self) -> "ScenarioResult": + if self.status != ScenarioStatus.PENDING: + raise RuntimeError( + "Cannot mark scenario as passed because its status has already been set") self._status = ScenarioStatus.PASSED return self @@ -38,6 +41,9 @@ def is_passed(self) -> bool: return self._status == ScenarioStatus.PASSED def mark_failed(self) -> "ScenarioResult": + if self.status != ScenarioStatus.PENDING: + raise RuntimeError( + "Cannot mark scenario as failed because its status has already been set") self._status = ScenarioStatus.FAILED return self @@ -45,6 +51,9 @@ def is_failed(self) -> bool: return self._status == ScenarioStatus.FAILED def mark_skipped(self) -> "ScenarioResult": + if self.status != ScenarioStatus.PENDING: + raise RuntimeError( + "Cannot mark scenario as skipped because its status has already been set") self._status = ScenarioStatus.SKIPPED return self @@ -90,7 +99,8 @@ def scope(self) -> ScopeType: return self._scope def attach(self, artifact: Artifact) -> None: - assert isinstance(artifact, Artifact) + if not isinstance(artifact, Artifact): + raise TypeError("artifact must be an instance of Artifact") self._artifacts.append(artifact) @property From d070a6b6f1172ec44dac061225d3e4c89c8119b4 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sun, 3 Dec 2023 23:09:26 +0400 Subject: [PATCH 04/13] add ff on repeat --- tests/plugins/repeater/_utils.py | 10 ++++++-- vedro/plugins/repeater/__init__.py | 4 +-- vedro/plugins/repeater/_repeater.py | 39 ++++++++++++++++++++++++----- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/tests/plugins/repeater/_utils.py b/tests/plugins/repeater/_utils.py index f0ded49e..4abaa04d 100644 --- a/tests/plugins/repeater/_utils.py +++ b/tests/plugins/repeater/_utils.py @@ -80,14 +80,20 @@ class Registry(Config.Registry): async def fire_arg_parsed_event(dispatcher: Dispatcher, *, - repeats: int, repeats_delay: float = 0.0) -> None: + repeats: int, + repeats_delay: float = 0.0, + fail_fast_on_repeat: bool = False) -> None: config_loaded_event = ConfigLoadedEvent(Path(), make_config()) await dispatcher.fire(config_loaded_event) arg_parse_event = ArgParseEvent(ArgumentParser()) await dispatcher.fire(arg_parse_event) - arg_parsed_event = ArgParsedEvent(Namespace(repeats=repeats, repeats_delay=repeats_delay)) + arg_parsed_event = ArgParsedEvent(Namespace( + repeats=repeats, + repeats_delay=repeats_delay, + fail_fast_on_repeat=fail_fast_on_repeat, + )) await dispatcher.fire(arg_parsed_event) diff --git a/vedro/plugins/repeater/__init__.py b/vedro/plugins/repeater/__init__.py index 6cb5f10a..c291e690 100644 --- a/vedro/plugins/repeater/__init__.py +++ b/vedro/plugins/repeater/__init__.py @@ -1,4 +1,4 @@ -from ._repeater import Repeater, RepeaterPlugin +from ._repeater import Repeater, RepeaterPlugin, RepeaterPluginTriggered from ._scheduler import RepeaterScenarioScheduler -__all__ = ("Repeater", "RepeaterPlugin", "RepeaterScenarioScheduler",) +__all__ = ("Repeater", "RepeaterPlugin", "RepeaterScenarioScheduler", "RepeaterPluginTriggered",) diff --git a/vedro/plugins/repeater/_repeater.py b/vedro/plugins/repeater/_repeater.py index fb42bef0..c02885a0 100644 --- a/vedro/plugins/repeater/_repeater.py +++ b/vedro/plugins/repeater/_repeater.py @@ -2,6 +2,7 @@ from typing import Any, Callable, Coroutine, Type, Union from vedro.core import ConfigType, Dispatcher, Plugin, PluginConfig, ScenarioScheduler +from vedro.core.scenario_runner import Interrupted from vedro.events import ( ArgParsedEvent, ArgParseEvent, @@ -9,12 +10,18 @@ ConfigLoadedEvent, ScenarioFailedEvent, ScenarioPassedEvent, + ScenarioRunEvent, + ScenarioSkippedEvent, StartupEvent, ) from ._scheduler import RepeaterScenarioScheduler -__all__ = ("Repeater", "RepeaterPlugin",) +__all__ = ("Repeater", "RepeaterPlugin", "RepeaterPluginTriggered",) + + +class RepeaterPluginTriggered(Interrupted): + pass SleepType = Callable[[float], Coroutine[Any, Any, None]] @@ -27,16 +34,20 @@ def __init__(self, config: Type["Repeater"], *, sleep: SleepType = asyncio.sleep self._sleep = sleep self._repeats: int = 1 self._repeats_delay: float = 0.0 + self._fail_fast: bool = False self._global_config: Union[ConfigType, None] = None self._scheduler: Union[ScenarioScheduler, None] = None self._repeat_scenario_id: Union[str, None] = None self._repeat_count: int = 0 + self._failed_count: int = 0 def subscribe(self, dispatcher: Dispatcher) -> None: dispatcher.listen(ConfigLoadedEvent, self.on_config_loaded) \ .listen(ArgParseEvent, self.on_arg_parse) \ .listen(ArgParsedEvent, self.on_arg_parsed) \ .listen(StartupEvent, self.on_startup) \ + .listen(ScenarioRunEvent, self.on_scenario_execute) \ + .listen(ScenarioSkippedEvent, self.on_scenario_execute) \ .listen(ScenarioPassedEvent, self.on_scenario_end) \ .listen(ScenarioFailedEvent, self.on_scenario_end) \ .listen(CleanupEvent, self.on_cleanup) @@ -51,10 +62,13 @@ def on_arg_parse(self, event: ArgParseEvent) -> None: group.add_argument("-N", "--repeats", type=int, default=self._repeats, help=help_message) group.add_argument("--repeats-delay", type=float, default=self._repeats_delay, help="Delay in seconds between scenario repeats (default: 0.0s)") + group.add_argument("--fail-fast-on-repeat", action="store_true", default=self._fail_fast, + help="Stop repeating scenarios after the first failure") def on_arg_parsed(self, event: ArgParsedEvent) -> None: self._repeats = event.args.repeats self._repeats_delay = event.args.repeats_delay + self._fail_fast = event.args.fail_fast_on_repeat if self._repeats < 1: raise ValueError("--repeats must be >= 1") @@ -65,6 +79,9 @@ def on_arg_parsed(self, event: ArgParsedEvent) -> None: if self._repeats_delay > 0.0 and self._repeats <= 1: raise ValueError("--repeats-delay must be used with --repeats > 1") + if self._fail_fast and self._repeats <= 1: + raise ValueError("--fail-fast-on-repeat must be used with --repeats > 1") + if self._repeats <= 1: return @@ -74,25 +91,35 @@ def on_arg_parsed(self, event: ArgParsedEvent) -> None: def on_startup(self, event: StartupEvent) -> None: self._scheduler = event.scheduler + def on_scenario_execute(self, event: Union[ScenarioRunEvent, ScenarioSkippedEvent]) -> None: + if not self._fail_fast: + return + + if self._fail_fast and self._failed_count >= 1: + raise RepeaterPluginTriggered("Stop repeating scenarios after the first failure") + async def on_scenario_end(self, event: Union[ScenarioPassedEvent, ScenarioFailedEvent]) -> None: if self._repeats <= 1: return assert isinstance(self._scheduler, RepeaterScenarioScheduler) # for type checking - scenario_id = event.scenario_result.scenario.unique_id - if self._repeat_scenario_id != scenario_id: - self._repeat_scenario_id = scenario_id + scenario = event.scenario_result.scenario + if self._repeat_scenario_id != scenario.unique_id: + self._repeat_scenario_id = scenario.unique_id self._repeat_count = 1 - self._scheduler.schedule(event.scenario_result.scenario) + self._scheduler.schedule(scenario) else: self._repeat_count += 1 if self._repeat_count < self._repeats: - self._scheduler.schedule(event.scenario_result.scenario) + self._scheduler.schedule(scenario) if self._repeats_delay > 0.0 and self._repeat_count < self._repeats: await self._sleep(self._repeats_delay) + if event.scenario_result.is_failed(): + self._failed_count = 1 + def on_cleanup(self, event: CleanupEvent) -> None: if self._repeats <= 1: return From da8ef3d77ee6b6c64eb16851887eafa60179d5af Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Tue, 9 Jan 2024 22:15:58 +0400 Subject: [PATCH 05/13] fix typo --- tests/plugins/repeater/test_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/plugins/repeater/test_scheduler.py b/tests/plugins/repeater/test_scheduler.py index 39d3512c..a6d2b6c3 100644 --- a/tests/plugins/repeater/test_scheduler.py +++ b/tests/plugins/repeater/test_scheduler.py @@ -33,8 +33,8 @@ def test_aggregate_nothing(scheduler: Scheduler): lambda: [make_scenario_result().mark_failed(), make_scenario_result().mark_failed()], lambda: [make_scenario_result().mark_skipped(), make_scenario_result().mark_skipped()], ]) -def test_aggreate_results(get_scenario_results: Callable[[], List[ScenarioResult]], *, - scheduler: Scheduler): +def test_aggregate_results(get_scenario_results: Callable[[], List[ScenarioResult]], *, + scheduler: Scheduler): with when: scenario_results = get_scenario_results() aggregated_result = scheduler.aggregate_results(scenario_results) From baae2ec087adc6613ec602af029e97411b7e5385 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Tue, 9 Jan 2024 22:16:07 +0400 Subject: [PATCH 06/13] update comment --- vedro/plugins/director/rich/_rich_reporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vedro/plugins/director/rich/_rich_reporter.py b/vedro/plugins/director/rich/_rich_reporter.py index d9797859..173dea36 100644 --- a/vedro/plugins/director/rich/_rich_reporter.py +++ b/vedro/plugins/director/rich/_rich_reporter.py @@ -321,7 +321,7 @@ class RichReporter(PluginConfig): # Show the elapsed time of each scenario show_timings: bool = False - # Show the relative path of each passed scenario + # Show the relative path of each executed scenario (passed, failed, or skipped) # Available if `show_scenario_extras` is True show_paths: bool = False From 2cd360bdb742b94414ccb44076a29a0ae42f249f Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Tue, 9 Jan 2024 22:43:59 +0400 Subject: [PATCH 07/13] update repeater --- .../core/scenario_runner/test_run_scenario.py | 2 +- vedro/plugins/repeater/__init__.py | 5 ++- vedro/plugins/repeater/_repeater.py | 39 ++++++++++++------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/core/scenario_runner/test_run_scenario.py b/tests/core/scenario_runner/test_run_scenario.py index 34cd32bc..9dd3ddd4 100644 --- a/tests/core/scenario_runner/test_run_scenario.py +++ b/tests/core/scenario_runner/test_run_scenario.py @@ -181,7 +181,7 @@ async def test_multiple_steps_failed(*, runner: MonotonicScenarioRunner, dispatc @pytest.mark.parametrize("interrupt_exception", (KeyboardInterrupt, Interrupted)) -async def test_step_interruped(interrupt_exception: Type[BaseException], *, dispatcher_: Mock): +async def test_step_interrupted(interrupt_exception: Type[BaseException], *, dispatcher_: Mock): with given: exception = interrupt_exception() step1_, step2_ = Mock(side_effect=exception), Mock(return_value=None) diff --git a/vedro/plugins/repeater/__init__.py b/vedro/plugins/repeater/__init__.py index c291e690..8e022dc8 100644 --- a/vedro/plugins/repeater/__init__.py +++ b/vedro/plugins/repeater/__init__.py @@ -1,4 +1,5 @@ -from ._repeater import Repeater, RepeaterPlugin, RepeaterPluginTriggered +from ._repeater import Repeater, RepeaterPlugin, RepeaterExecutionInterrupted from ._scheduler import RepeaterScenarioScheduler -__all__ = ("Repeater", "RepeaterPlugin", "RepeaterScenarioScheduler", "RepeaterPluginTriggered",) +__all__ = ("Repeater", "RepeaterPlugin", "RepeaterScenarioScheduler", + "RepeaterExecutionInterrupted",) diff --git a/vedro/plugins/repeater/_repeater.py b/vedro/plugins/repeater/_repeater.py index c02885a0..c6cb2e44 100644 --- a/vedro/plugins/repeater/_repeater.py +++ b/vedro/plugins/repeater/_repeater.py @@ -17,10 +17,18 @@ from ._scheduler import RepeaterScenarioScheduler -__all__ = ("Repeater", "RepeaterPlugin", "RepeaterPluginTriggered",) +__all__ = ("Repeater", "RepeaterPlugin", "RepeaterExecutionInterrupted",) -class RepeaterPluginTriggered(Interrupted): +class RepeaterExecutionInterrupted(Interrupted): + """ + Exception raised when the execution of scenario repetition is interrupted. + + This exception is used within the RepeaterPlugin to signal an early termination + of the scenario repetition process. It is typically raised when the fail-fast + condition is met, i.e., if a scenario fails and the --fail-fast-on-repeat option + is enabled, indicating that further repetitions of the scenario should be stopped. + """ pass @@ -76,13 +84,13 @@ def on_arg_parsed(self, event: ArgParsedEvent) -> None: if self._repeats_delay < 0.0: raise ValueError("--repeats-delay must be >= 0.0") - if self._repeats_delay > 0.0 and self._repeats <= 1: + if (self._repeats_delay > 0.0) and (self._repeats <= 1): raise ValueError("--repeats-delay must be used with --repeats > 1") - if self._fail_fast and self._repeats <= 1: + if self._fail_fast and (self._repeats <= 1): raise ValueError("--fail-fast-on-repeat must be used with --repeats > 1") - if self._repeats <= 1: + if not self.is_repeating_enabled(): return assert self._global_config is not None # for type checking @@ -92,15 +100,12 @@ def on_startup(self, event: StartupEvent) -> None: self._scheduler = event.scheduler def on_scenario_execute(self, event: Union[ScenarioRunEvent, ScenarioSkippedEvent]) -> None: - if not self._fail_fast: - return - - if self._fail_fast and self._failed_count >= 1: - raise RepeaterPluginTriggered("Stop repeating scenarios after the first failure") + if self._fail_fast and (self._failed_count >= 1): + raise RepeaterExecutionInterrupted("Stop repeating scenarios after the first failure") async def on_scenario_end(self, event: Union[ScenarioPassedEvent, ScenarioFailedEvent]) -> None: - if self._repeats <= 1: + if not self.is_repeating_enabled(): return assert isinstance(self._scheduler, RepeaterScenarioScheduler) # for type checking @@ -114,17 +119,21 @@ async def on_scenario_end(self, if self._repeat_count < self._repeats: self._scheduler.schedule(scenario) - if self._repeats_delay > 0.0 and self._repeat_count < self._repeats: + if (self._repeats_delay > 0.0) and (self._repeat_count < self._repeats): await self._sleep(self._repeats_delay) if event.scenario_result.is_failed(): self._failed_count = 1 def on_cleanup(self, event: CleanupEvent) -> None: - if self._repeats <= 1: + if not self.is_repeating_enabled(): return - message = self._get_summary_message() - event.report.add_summary(message) + if not self._fail_fast or (self._failed_count == 0): + message = self._get_summary_message() + event.report.add_summary(message) + + def is_repeating_enabled(self) -> bool: + return self._repeats > 1 def _get_summary_message(self) -> str: message = f"repeated x{self._repeats}" From 8a02f5ae0c59dc3756abe6c1a7e771a246d07355 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 13:24:27 +0400 Subject: [PATCH 08/13] update repeater --- tests/plugins/repeater/test_repeater.py | 74 ++++--------------------- vedro/plugins/repeater/__init__.py | 2 +- vedro/plugins/repeater/_repeater.py | 48 ++++++++-------- 3 files changed, 36 insertions(+), 88 deletions(-) diff --git a/tests/plugins/repeater/test_repeater.py b/tests/plugins/repeater/test_repeater.py index f0ab4470..3802261d 100644 --- a/tests/plugins/repeater/test_repeater.py +++ b/tests/plugins/repeater/test_repeater.py @@ -17,7 +17,6 @@ dispatcher, fire_arg_parsed_event, fire_failed_event, - fire_passed_event, fire_startup_event, make_scenario_result, repeater, @@ -28,7 +27,7 @@ __all__ = ("dispatcher", "repeater", "scheduler_", "sleep_") # fixtures -@pytest.mark.parametrize("repeats", [2, 3]) +@pytest.mark.parametrize("repeats", [1, 2, 3]) @pytest.mark.parametrize("get_event", [ lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), @@ -47,58 +46,7 @@ async def test_repeat(repeats: int, get_event: Callable[[ScenarioResult], Event] await dispatcher.fire(scenario_event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] - assert sleep_.mock_calls == [] - - -@pytest.mark.parametrize("get_event", [ - lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), - lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), -]) -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat2_fired_twice(get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): - with given: - await fire_arg_parsed_event(dispatcher, repeats=2) - await fire_startup_event(dispatcher, scheduler_) - - scenario_result1 = await fire_passed_event(dispatcher) - scenario_result2 = make_scenario_result(scenario_result1.scenario) - - scenario_event = get_event(scenario_result2) - scheduler_.reset_mock() - - with when: - await dispatcher.fire(scenario_event) - - with then: - assert scheduler_.mock_calls == [] - assert sleep_.mock_calls == [] - - -@pytest.mark.parametrize("repeats", [3, 4]) -@pytest.mark.parametrize("get_event", [ - lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), - lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), -]) -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat3_fired_twice(repeats: int, get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): - with given: - await fire_arg_parsed_event(dispatcher, repeats=repeats) - await fire_startup_event(dispatcher, scheduler_) - - scenario_result1 = await fire_passed_event(dispatcher) - scenario_result2 = make_scenario_result(scenario_result1.scenario) - - scenario_event = get_event(scenario_result2) - scheduler_.reset_mock() - - with when: - await dispatcher.fire(scenario_event) - - with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result2.scenario)] + assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] * (repeats - 1) assert sleep_.mock_calls == [] @@ -126,8 +74,8 @@ async def test_dont_repeat_skipped(repeats: int, *, (3, 1.0) ]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_with_delay(repeats: int, repeats_delay: float, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def _test_repeat_with_delay(repeats: int, repeats_delay: float, *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=repeats, repeats_delay=repeats_delay) await fire_startup_event(dispatcher, scheduler_) @@ -148,9 +96,9 @@ async def test_repeat_with_delay(repeats: int, repeats_delay: float, *, lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), ]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat2_with_delay_fired_twice(get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, - scheduler_: Mock, sleep_: AsyncMock): +async def _test_repeat2_with_delay_fired_twice(get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, + scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=0.1) await fire_startup_event(dispatcher, scheduler_) @@ -176,10 +124,10 @@ async def test_repeat2_with_delay_fired_twice(get_event: Callable[[ScenarioResul lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), ]) @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat3_with_delay_fired_twice(repeats: int, - get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, - scheduler_: Mock, sleep_: AsyncMock): +async def _test_repeat3_with_delay_fired_twice(repeats: int, + get_event: Callable[[ScenarioResult], Event], *, + dispatcher: Dispatcher, + scheduler_: Mock, sleep_: AsyncMock): with given: repeats_delay = 0.1 await fire_arg_parsed_event(dispatcher, repeats=repeats, repeats_delay=repeats_delay) diff --git a/vedro/plugins/repeater/__init__.py b/vedro/plugins/repeater/__init__.py index 8e022dc8..0a5e30c1 100644 --- a/vedro/plugins/repeater/__init__.py +++ b/vedro/plugins/repeater/__init__.py @@ -1,4 +1,4 @@ -from ._repeater import Repeater, RepeaterPlugin, RepeaterExecutionInterrupted +from ._repeater import Repeater, RepeaterExecutionInterrupted, RepeaterPlugin from ._scheduler import RepeaterScenarioScheduler __all__ = ("Repeater", "RepeaterPlugin", "RepeaterScenarioScheduler", diff --git a/vedro/plugins/repeater/_repeater.py b/vedro/plugins/repeater/_repeater.py index c6cb2e44..4f2270f2 100644 --- a/vedro/plugins/repeater/_repeater.py +++ b/vedro/plugins/repeater/_repeater.py @@ -46,7 +46,6 @@ def __init__(self, config: Type["Repeater"], *, sleep: SleepType = asyncio.sleep self._global_config: Union[ConfigType, None] = None self._scheduler: Union[ScenarioScheduler, None] = None self._repeat_scenario_id: Union[str, None] = None - self._repeat_count: int = 0 self._failed_count: int = 0 def subscribe(self, dispatcher: Dispatcher) -> None: @@ -90,49 +89,50 @@ def on_arg_parsed(self, event: ArgParsedEvent) -> None: if self._fail_fast and (self._repeats <= 1): raise ValueError("--fail-fast-on-repeat must be used with --repeats > 1") - if not self.is_repeating_enabled(): - return - - assert self._global_config is not None # for type checking - self._global_config.Registry.ScenarioScheduler.register(self._scheduler_factory, self) + if self._is_repeating_enabled(): + assert self._global_config is not None # for type checking + self._global_config.Registry.ScenarioScheduler.register(self._scheduler_factory, self) def on_startup(self, event: StartupEvent) -> None: self._scheduler = event.scheduler - def on_scenario_execute(self, event: Union[ScenarioRunEvent, ScenarioSkippedEvent]) -> None: - if self._fail_fast and (self._failed_count >= 1): + async def on_scenario_execute(self, + event: Union[ScenarioRunEvent, ScenarioSkippedEvent]) -> None: + if not self._is_repeating_enabled(): + return + + if self._fail_fast and (self._failed_count > 0): raise RepeaterExecutionInterrupted("Stop repeating scenarios after the first failure") + scenario = event.scenario_result.scenario + if (self._repeat_scenario_id == scenario.unique_id) and (self._repeats_delay > 0.0): + await self._sleep(self._repeats_delay) + async def on_scenario_end(self, event: Union[ScenarioPassedEvent, ScenarioFailedEvent]) -> None: - if not self.is_repeating_enabled(): + if not self._is_repeating_enabled(): return assert isinstance(self._scheduler, RepeaterScenarioScheduler) # for type checking scenario = event.scenario_result.scenario - if self._repeat_scenario_id != scenario.unique_id: + if scenario.unique_id != self._repeat_scenario_id: self._repeat_scenario_id = scenario.unique_id - self._repeat_count = 1 - self._scheduler.schedule(scenario) - else: - self._repeat_count += 1 - if self._repeat_count < self._repeats: + self._failed_count = 1 if event.scenario_result.is_failed() else 0 + for _ in range(self._repeats - 1): self._scheduler.schedule(scenario) - - if (self._repeats_delay > 0.0) and (self._repeat_count < self._repeats): - await self._sleep(self._repeats_delay) - - if event.scenario_result.is_failed(): - self._failed_count = 1 + else: + if event.scenario_result.is_failed(): + self._failed_count += 1 def on_cleanup(self, event: CleanupEvent) -> None: - if not self.is_repeating_enabled(): + if not self._is_repeating_enabled(): return - if not self._fail_fast or (self._failed_count == 0): + + if not event.report.interrupted: message = self._get_summary_message() event.report.add_summary(message) - def is_repeating_enabled(self) -> bool: + def _is_repeating_enabled(self) -> bool: return self._repeats > 1 def _get_summary_message(self) -> str: From 6e705c3d8d2b9bd6ee89eb4669ca30c8c39bac22 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 13:34:18 +0400 Subject: [PATCH 09/13] update rerunner --- tests/plugins/rerunner/test_rerunner.py | 7 +-- vedro/plugins/rerunner/_rerunner.py | 67 ++++++++++++++++--------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/tests/plugins/rerunner/test_rerunner.py b/tests/plugins/rerunner/test_rerunner.py index 1f6d3514..ceb3a415 100644 --- a/tests/plugins/rerunner/test_rerunner.py +++ b/tests/plugins/rerunner/test_rerunner.py @@ -114,7 +114,8 @@ async def test_dont_rerun_skipped(reruns: int, *, @pytest.mark.usefixtures(rerunner.__name__) -async def test_dont_rerun_rerunned(dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_no_additional_reruns_after_failure(dispatcher: Dispatcher, + scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, reruns=1) await fire_startup_event(dispatcher, scheduler_) @@ -135,8 +136,8 @@ async def test_dont_rerun_rerunned(dispatcher: Dispatcher, scheduler_: Mock, sle (2, 1.0) ]) @pytest.mark.usefixtures(rerunner.__name__) -async def test_rerun_with_delay(reruns: int, reruns_delay: float, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def _test_rerun_with_delay(reruns: int, reruns_delay: float, *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, reruns=reruns, reruns_delay=reruns_delay) await fire_startup_event(dispatcher, scheduler_) diff --git a/vedro/plugins/rerunner/_rerunner.py b/vedro/plugins/rerunner/_rerunner.py index 9f4ecbe7..5a3c886e 100644 --- a/vedro/plugins/rerunner/_rerunner.py +++ b/vedro/plugins/rerunner/_rerunner.py @@ -9,6 +9,8 @@ ConfigLoadedEvent, ScenarioFailedEvent, ScenarioPassedEvent, + ScenarioRunEvent, + ScenarioSkippedEvent, StartupEvent, ) @@ -37,6 +39,8 @@ def subscribe(self, dispatcher: Dispatcher) -> None: .listen(ArgParseEvent, self.on_arg_parse) \ .listen(ArgParsedEvent, self.on_arg_parsed) \ .listen(StartupEvent, self.on_startup) \ + .listen(ScenarioRunEvent, self.on_scenario_execute) \ + .listen(ScenarioSkippedEvent, self.on_scenario_execute) \ .listen(ScenarioPassedEvent, self.on_scenario_end) \ .listen(ScenarioFailedEvent, self.on_scenario_end) \ .listen(CleanupEvent, self.on_cleanup) @@ -62,45 +66,58 @@ def on_arg_parsed(self, event: ArgParsedEvent) -> None: if self._reruns_delay < 0.0: raise ValueError("--reruns-delay must be >= 0.0") - if self._reruns_delay > 0.0 and self._reruns < 1: + if (self._reruns_delay > 0.0) and (self._reruns < 1): raise ValueError("--reruns-delay must be used with --reruns > 0") - if self._reruns == 0: - return - - assert self._global_config is not None # for type checking - self._global_config.Registry.ScenarioScheduler.register(self._scheduler_factory, self) + if self._is_rerunning_enabled(): + assert self._global_config is not None # for type checking + self._global_config.Registry.ScenarioScheduler.register(self._scheduler_factory, self) def on_startup(self, event: StartupEvent) -> None: self._scheduler = event.scheduler + async def on_scenario_execute(self, + event: Union[ScenarioRunEvent, ScenarioSkippedEvent]) -> None: + if not self._is_rerunning_enabled(): + return + + scenario = event.scenario_result.scenario + if (self._rerun_scenario_id == scenario.unique_id) and (self._reruns_delay > 0.0): + await self._sleep(self._reruns_delay) + async def on_scenario_end(self, event: Union[ScenarioPassedEvent, ScenarioFailedEvent]) -> None: - if self._reruns == 0: + if not self._is_rerunning_enabled(): return - assert isinstance(self._scheduler, RerunnerScenarioScheduler) # for type checking - if self._rerun_scenario_id == event.scenario_result.scenario.unique_id: - return + scenario = event.scenario_result.scenario + if scenario.unique_id != self._rerun_scenario_id: + self._rerun_scenario_id = scenario.unique_id - self._rerun_scenario_id = event.scenario_result.scenario.unique_id - if event.scenario_result.is_failed(): - self._reran += 1 - for _ in range(self._reruns): - if self._reruns_delay > 0.0: - await self._sleep(self._reruns_delay) - self._scheduler.schedule(event.scenario_result.scenario) - self._times += 1 + if event.scenario_result.is_failed(): + self._reran += 1 + for _ in range(self._reruns): + self._scheduler.schedule(scenario) + self._times += 1 def on_cleanup(self, event: CleanupEvent) -> None: - if self._reruns != 0: - ss = "" if self._reran == 1 else "s" - ts = "" if self._times == 1 else "s" - message = f"rerun {self._reran} scenario{ss}, {self._times} time{ts}" - if self._reruns_delay: - message += f", with delay {self._reruns_delay!r}s" - event.report.add_summary(message) + if not self._is_rerunning_enabled(): + return + + message = self._get_summary_message() + event.report.add_summary(message) + + def _is_rerunning_enabled(self) -> bool: + return self._reruns > 0 + + def _get_summary_message(self) -> str: + ss = "" if self._reran == 1 else "s" + ts = "" if self._times == 1 else "s" + message = f"rerun {self._reran} scenario{ss}, {self._times} time{ts}" + if self._reruns_delay: + message += f", with delay {self._reruns_delay!r}s" + return message class Rerunner(PluginConfig): From a263a2cb64c24f9fe5ea2d6a6ada7b72383529f9 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 13:49:00 +0400 Subject: [PATCH 10/13] fix elapsed --- .../scenario_result/test_aggregated_result.py | 38 +++++++++++++++++++ .../scenario_result/_aggregated_result.py | 12 +++--- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/core/scenario_result/test_aggregated_result.py b/tests/core/scenario_result/test_aggregated_result.py index e3313f89..fdbe853c 100644 --- a/tests/core/scenario_result/test_aggregated_result.py +++ b/tests/core/scenario_result/test_aggregated_result.py @@ -115,6 +115,44 @@ def test_from_existing_started_ended(get_scenario_result: Callable[[], ScenarioR assert aggregated_result.extra_details == scenario_result.extra_details +def test_from_existing_started_at_min(): + with given: + scenario_result1 = make_scenario_result().set_started_at(1.0) + scenario_result2 = make_scenario_result().set_started_at(2.0) + + with when: + aggregated_result = AggregatedResult.from_existing(scenario_result2, + [scenario_result1, scenario_result2]) + + with then: + assert aggregated_result.status == scenario_result2.status + assert aggregated_result.started_at == scenario_result1.started_at # min + assert aggregated_result.ended_at == scenario_result2.ended_at + assert aggregated_result.scope == scenario_result2.scope + assert aggregated_result.artifacts == scenario_result2.artifacts + assert aggregated_result.step_results == scenario_result2.step_results + assert aggregated_result.extra_details == scenario_result2.extra_details + + +def test_from_existing_ended_at_max(): + with given: + scenario_result1 = make_scenario_result().set_ended_at(1.0) + scenario_result2 = make_scenario_result().set_ended_at(5.0) + + with when: + aggregated_result = AggregatedResult.from_existing(scenario_result1, + [scenario_result1, scenario_result2]) + + with then: + assert aggregated_result.status == scenario_result1.status + assert aggregated_result.started_at == scenario_result1.started_at + assert aggregated_result.ended_at == scenario_result2.ended_at # max + assert aggregated_result.scope == scenario_result1.scope + assert aggregated_result.artifacts == scenario_result1.artifacts + assert aggregated_result.step_results == scenario_result1.step_results + assert aggregated_result.extra_details == scenario_result1.extra_details + + def test_from_existing_artifacts(): with given: scenario_result = make_scenario_result() diff --git a/vedro/core/scenario_result/_aggregated_result.py b/vedro/core/scenario_result/_aggregated_result.py index ea86bcc3..26bd950a 100644 --- a/vedro/core/scenario_result/_aggregated_result.py +++ b/vedro/core/scenario_result/_aggregated_result.py @@ -31,11 +31,6 @@ def from_existing(main_scenario_result: ScenarioResult, elif main_scenario_result.is_skipped(): result.mark_skipped() - if main_scenario_result.started_at is not None: - result.set_started_at(main_scenario_result.started_at) - if main_scenario_result.ended_at is not None: - result.set_ended_at(main_scenario_result.ended_at) - result.set_scope(main_scenario_result.scope) for step_result in main_scenario_result.step_results: @@ -49,6 +44,13 @@ def from_existing(main_scenario_result: ScenarioResult, assert len(scenario_results) > 0 for scenario_result in scenario_results: + if scenario_result.started_at is not None: + if (result.started_at is None) or (scenario_result.started_at < result.started_at): + result.set_started_at(scenario_result.started_at) + if scenario_result.ended_at is not None: + if (result.ended_at is None) or (scenario_result.ended_at > result.ended_at): + result.set_ended_at(scenario_result.ended_at) + result.add_scenario_result(scenario_result) return result From 90231cfe73ad12656f754570ae09d8b83c7fd594 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 17:41:59 +0400 Subject: [PATCH 11/13] remove filter_internals --- vedro/plugins/director/rich/_rich_printer.py | 32 ++++++++++++++++--- vedro/plugins/director/rich/utils/__init__.py | 3 -- .../director/rich/utils/_filter_internals.py | 32 ------------------- 3 files changed, 27 insertions(+), 40 deletions(-) delete mode 100644 vedro/plugins/director/rich/utils/_filter_internals.py diff --git a/vedro/plugins/director/rich/_rich_printer.py b/vedro/plugins/director/rich/_rich_printer.py index d23e0bc8..f8c79964 100644 --- a/vedro/plugins/director/rich/_rich_printer.py +++ b/vedro/plugins/director/rich/_rich_printer.py @@ -1,8 +1,10 @@ import json +import os import warnings from os import linesep from traceback import format_exception -from typing import Any, Callable, Dict, List, Optional, Union +from types import FrameType, TracebackType +from typing import Any, Callable, Dict, List, Optional, Union, cast from rich.console import Console, RenderableType from rich.pretty import Pretty @@ -10,10 +12,9 @@ from rich.style import Style from rich.traceback import Trace, Traceback +import vedro from vedro.core import ExcInfo, ScenarioStatus, StepStatus -from .utils import filter_internals - __all__ = ("RichPrinter",) @@ -92,12 +93,33 @@ def print_step_name(self, name: str, status: StepStatus, *, else: self._console.out(name, style=style) + def __filter_internals(self, traceback: TracebackType) -> TracebackType: + class _Traceback: + def __init__(self, tb_frame: FrameType, tb_lasti: int, tb_lineno: int, + tb_next: Optional[TracebackType]) -> None: + self.tb_frame = tb_frame + self.tb_lasti = tb_lasti + self.tb_lineno = tb_lineno + self.tb_next = tb_next + + tb = _Traceback(traceback.tb_frame, traceback.tb_lasti, traceback.tb_lineno, + traceback.tb_next) + + root = os.path.dirname(vedro.__file__) + while tb.tb_next is not None: + filename = os.path.abspath(tb.tb_frame.f_code.co_filename) + if os.path.commonpath([root, filename]) != root: + break + tb = tb.tb_next # type: ignore + + return cast(TracebackType, tb) + def print_exception(self, exc_info: ExcInfo, *, max_frames: int = 8, show_internal_calls: bool = False) -> None: if show_internal_calls: traceback = exc_info.traceback else: - traceback = filter_internals(exc_info.traceback) + traceback = self.__filter_internals(exc_info.traceback) formatted = format_exception(exc_info.type, exc_info.value, traceback, limit=max_frames) self._console.out("".join(formatted), style=Style(color="yellow")) @@ -117,7 +139,7 @@ def print_pretty_exception(self, exc_info: ExcInfo, *, if show_internal_calls: traceback = exc_info.traceback else: - traceback = filter_internals(exc_info.traceback) + traceback = self.__filter_internals(exc_info.traceback) trace = Traceback.extract(exc_info.type, exc_info.value, traceback, show_locals=show_locals) diff --git a/vedro/plugins/director/rich/utils/__init__.py b/vedro/plugins/director/rich/utils/__init__.py index 9c300b6e..e69de29b 100644 --- a/vedro/plugins/director/rich/utils/__init__.py +++ b/vedro/plugins/director/rich/utils/__init__.py @@ -1,3 +0,0 @@ -from ._filter_internals import filter_internals - -__all__ = ("filter_internals",) diff --git a/vedro/plugins/director/rich/utils/_filter_internals.py b/vedro/plugins/director/rich/utils/_filter_internals.py deleted file mode 100644 index 833ead3f..00000000 --- a/vedro/plugins/director/rich/utils/_filter_internals.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -from types import FrameType, TracebackType -from typing import Optional, cast - -import vedro - -__all__ = ("filter_internals",) - - -def filter_internals(traceback: TracebackType) -> TracebackType: - class _Traceback: - def __init__(self, - tb_frame: FrameType, - tb_lasti: int, - tb_lineno: int, - tb_next: Optional[TracebackType]) -> None: - self.tb_frame = tb_frame - self.tb_lasti = tb_lasti - self.tb_lineno = tb_lineno - self.tb_next = tb_next - - tb = _Traceback(traceback.tb_frame, traceback.tb_lasti, traceback.tb_lineno, - traceback.tb_next) - - root = os.path.dirname(vedro.__file__) - while tb.tb_next is not None: - filename = os.path.abspath(tb.tb_frame.f_code.co_filename) - if os.path.commonpath([root, filename]) != root: - break - tb = tb.tb_next # type: ignore - - return cast(TracebackType, tb) From 2a48fab30462e9a4bdf942eba0a76e6027de9d51 Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 17:58:21 +0400 Subject: [PATCH 12/13] update tests --- tests/plugins/rerunner/test_rerunner.py | 53 ++++++++++++++++++------- vedro/plugins/rerunner/_rerunner.py | 1 - 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/tests/plugins/rerunner/test_rerunner.py b/tests/plugins/rerunner/test_rerunner.py index ceb3a415..680f91ab 100644 --- a/tests/plugins/rerunner/test_rerunner.py +++ b/tests/plugins/rerunner/test_rerunner.py @@ -1,3 +1,4 @@ +from typing import Type, Union from unittest.mock import AsyncMock, Mock, call import pytest @@ -9,6 +10,7 @@ CleanupEvent, ScenarioFailedEvent, ScenarioPassedEvent, + ScenarioRunEvent, ScenarioSkippedEvent, ) @@ -27,7 +29,7 @@ @pytest.mark.usefixtures(rerunner.__name__) -async def test_rerun_validation(dispatcher: Dispatcher): +async def test_reruns_validation(dispatcher: Dispatcher): with when, raises(BaseException) as exc_info: await fire_arg_parsed_event(dispatcher, reruns=-1) @@ -37,9 +39,9 @@ async def test_rerun_validation(dispatcher: Dispatcher): @pytest.mark.usefixtures(rerunner.__name__) -async def test_rerun_delay_validation(dispatcher: Dispatcher): +async def test_reruns_delay_validation(dispatcher: Dispatcher): with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, reruns=1, reruns_delay=-0.001) + await fire_arg_parsed_event(dispatcher, reruns=1, reruns_delay=-0.1) with then: assert exc_info.type is ValueError @@ -47,7 +49,7 @@ async def test_rerun_delay_validation(dispatcher: Dispatcher): @pytest.mark.usefixtures(rerunner.__name__) -async def test_rerun_delay_without_reruns_validation(dispatcher: Dispatcher): +async def test_reruns_delay_without_reruns_validation(dispatcher: Dispatcher): with when, raises(BaseException) as exc_info: await fire_arg_parsed_event(dispatcher, reruns=0, reruns_delay=0.1) @@ -131,26 +133,47 @@ async def test_no_additional_reruns_after_failure(dispatcher: Dispatcher, assert sleep_.mock_calls == [] -@pytest.mark.parametrize(("reruns", "reruns_delay"), [ - (1, 0.1), - (2, 1.0) +@pytest.mark.parametrize(("event_cls", "reruns_delay"), [ + (ScenarioRunEvent, 0.1), + (ScenarioSkippedEvent, 1.0) ]) @pytest.mark.usefixtures(rerunner.__name__) -async def _test_rerun_with_delay(reruns: int, reruns_delay: float, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_reruns_delay(event_cls: Union[Type[ScenarioRunEvent], Type[ScenarioSkippedEvent]], + reruns_delay: float, *, dispatcher: Dispatcher, scheduler_: Mock, + sleep_: AsyncMock): with given: - await fire_arg_parsed_event(dispatcher, reruns=reruns, reruns_delay=reruns_delay) + await fire_arg_parsed_event(dispatcher, reruns=2, reruns_delay=reruns_delay) await fire_startup_event(dispatcher, scheduler_) + scenario_failed_event = await fire_failed_event(dispatcher) + scheduler_.reset_mock() - scenario_result = make_scenario_result().mark_failed() - scenario_failed_event = ScenarioFailedEvent(scenario_result) + event = event_cls(scenario_failed_event.scenario_result) with when: - await dispatcher.fire(scenario_failed_event) + await dispatcher.fire(event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] * reruns - assert sleep_.mock_calls == [call(reruns_delay)] * reruns + assert scheduler_.mock_calls == [] + assert sleep_.mock_calls == [call(reruns_delay)] + + +@pytest.mark.usefixtures(rerunner.__name__) +async def test_no_reruns_delay(*, dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): + with given: + await fire_arg_parsed_event(dispatcher, reruns=2, reruns_delay=0.1) + await fire_startup_event(dispatcher, scheduler_) + await fire_failed_event(dispatcher) + scheduler_.reset_mock() + + scenario_result = make_scenario_result() + event = ScenarioRunEvent(scenario_result) + + with when: + await dispatcher.fire(event) + + with then: + assert scheduler_.mock_calls == [] + assert sleep_.mock_calls == [] @pytest.mark.usefixtures(rerunner.__name__) diff --git a/vedro/plugins/rerunner/_rerunner.py b/vedro/plugins/rerunner/_rerunner.py index 5a3c886e..73756416 100644 --- a/vedro/plugins/rerunner/_rerunner.py +++ b/vedro/plugins/rerunner/_rerunner.py @@ -104,7 +104,6 @@ async def on_scenario_end(self, def on_cleanup(self, event: CleanupEvent) -> None: if not self._is_rerunning_enabled(): return - message = self._get_summary_message() event.report.add_summary(message) From 1cc83d73b167a116362155998d543dc2a557394f Mon Sep 17 00:00:00 2001 From: Nikita Tsvetkov Date: Sat, 13 Jan 2024 18:17:50 +0400 Subject: [PATCH 13/13] update tests --- tests/plugins/repeater/_utils.py | 13 +- tests/plugins/repeater/test_repeater.py | 170 +++++++++++++++--------- 2 files changed, 116 insertions(+), 67 deletions(-) diff --git a/tests/plugins/repeater/_utils.py b/tests/plugins/repeater/_utils.py index 4abaa04d..c74abaab 100644 --- a/tests/plugins/repeater/_utils.py +++ b/tests/plugins/repeater/_utils.py @@ -1,8 +1,10 @@ import asyncio +import sys from argparse import ArgumentParser, Namespace from pathlib import Path from time import monotonic_ns -from typing import Optional +from types import TracebackType +from typing import Optional, cast from unittest.mock import AsyncMock, Mock import pytest @@ -12,6 +14,7 @@ Config, ConfigType, Dispatcher, + ExcInfo, Factory, MonotonicScenarioScheduler, ScenarioResult, @@ -114,3 +117,11 @@ async def fire_failed_event(dispatcher: Dispatcher) -> ScenarioResult: scenario_failed_event = ScenarioFailedEvent(scenario_result) await dispatcher.fire(scenario_failed_event) return scenario_result + + +def make_exc_info(exc_val: BaseException) -> ExcInfo: + try: + raise exc_val + except type(exc_val): + *_, traceback = sys.exc_info() + return ExcInfo(type(exc_val), exc_val, cast(TracebackType, traceback)) diff --git a/tests/plugins/repeater/test_repeater.py b/tests/plugins/repeater/test_repeater.py index 3802261d..4eaaa278 100644 --- a/tests/plugins/repeater/test_repeater.py +++ b/tests/plugins/repeater/test_repeater.py @@ -1,4 +1,4 @@ -from typing import Callable +from typing import Callable, Type, Union from unittest.mock import AsyncMock, Mock, call import pytest @@ -10,14 +10,18 @@ CleanupEvent, ScenarioFailedEvent, ScenarioPassedEvent, + ScenarioRunEvent, ScenarioSkippedEvent, ) +from vedro.plugins.repeater import RepeaterExecutionInterrupted from ._utils import ( dispatcher, fire_arg_parsed_event, fire_failed_event, + fire_passed_event, fire_startup_event, + make_exc_info, make_scenario_result, repeater, scheduler_, @@ -27,6 +31,49 @@ __all__ = ("dispatcher", "repeater", "scheduler_", "sleep_") # fixtures +ScenarioExecuteFactory = Union[Type[ScenarioRunEvent], Type[ScenarioSkippedEvent]] + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeats_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=0) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--repeats must be >= 1" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeats_delay_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=-0.001) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--repeats-delay must be >= 0.0" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeats_delay_without_repeats_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=1, repeats_delay=0.1) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--repeats-delay must be used with --repeats > 1" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_fail_fast_without_repeats_validation(dispatcher: Dispatcher): + with when, raises(BaseException) as exc_info: + await fire_arg_parsed_event(dispatcher, repeats=1, fail_fast_on_repeat=True) + + with then: + assert exc_info.type is ValueError + assert str(exc_info.value) == "--fail-fast-on-repeat must be used with --repeats > 1" + + @pytest.mark.parametrize("repeats", [1, 2, 3]) @pytest.mark.parametrize("get_event", [ lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), @@ -69,83 +116,84 @@ async def test_dont_repeat_skipped(repeats: int, *, assert sleep_.mock_calls == [] -@pytest.mark.parametrize(("repeats", "repeats_delay"), [ - (2, 0.1), - (3, 1.0) +@pytest.mark.parametrize(("event_cls", "repeats_delay"), [ + (ScenarioRunEvent, 0.1), + (ScenarioSkippedEvent, 1.0) ]) @pytest.mark.usefixtures(repeater.__name__) -async def _test_repeat_with_delay(repeats: int, repeats_delay: float, *, - dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): +async def test_repeats_delay(event_cls: ScenarioExecuteFactory, repeats_delay: float, *, + dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: - await fire_arg_parsed_event(dispatcher, repeats=repeats, repeats_delay=repeats_delay) + await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=repeats_delay) await fire_startup_event(dispatcher, scheduler_) - scenario_result = make_scenario_result().mark_passed() - scenario_passed_event = ScenarioPassedEvent(scenario_result) + scenario_result = await fire_failed_event(dispatcher) + scheduler_.reset_mock() + + event = event_cls(scenario_result) with when: - await dispatcher.fire(scenario_passed_event) + await dispatcher.fire(event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result.scenario)] + assert scheduler_.mock_calls == [] assert sleep_.mock_calls == [call(repeats_delay)] -@pytest.mark.parametrize("get_event", [ - lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), - lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), -]) @pytest.mark.usefixtures(repeater.__name__) -async def _test_repeat2_with_delay_fired_twice(get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, - scheduler_: Mock, sleep_: AsyncMock): +async def test_no_repeats_delay(*, dispatcher: Dispatcher, scheduler_: Mock, sleep_: AsyncMock): with given: await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=0.1) await fire_startup_event(dispatcher, scheduler_) - - scenario_result1 = await fire_failed_event(dispatcher) - scenario_result2 = make_scenario_result(scenario_result1.scenario) - - scenario_event = get_event(scenario_result2) + await fire_failed_event(dispatcher) scheduler_.reset_mock() - sleep_.reset_mock() + + scenario_result = make_scenario_result() + event = ScenarioRunEvent(scenario_result) with when: - await dispatcher.fire(scenario_event) + await dispatcher.fire(event) with then: assert scheduler_.mock_calls == [] assert sleep_.mock_calls == [] -@pytest.mark.parametrize("repeats", [3, 4]) -@pytest.mark.parametrize("get_event", [ - lambda scn_result: ScenarioPassedEvent(scn_result.mark_passed()), - lambda scn_result: ScenarioFailedEvent(scn_result.mark_failed()), -]) +@pytest.mark.parametrize("event_cls", [ScenarioRunEvent, ScenarioSkippedEvent]) @pytest.mark.usefixtures(repeater.__name__) -async def _test_repeat3_with_delay_fired_twice(repeats: int, - get_event: Callable[[ScenarioResult], Event], *, - dispatcher: Dispatcher, - scheduler_: Mock, sleep_: AsyncMock): +async def test_repeats_fail_fast(event_cls: ScenarioExecuteFactory, *, dispatcher: Dispatcher, + scheduler_: Mock): with given: - repeats_delay = 0.1 - await fire_arg_parsed_event(dispatcher, repeats=repeats, repeats_delay=repeats_delay) + await fire_arg_parsed_event(dispatcher, repeats=2, fail_fast_on_repeat=True) await fire_startup_event(dispatcher, scheduler_) + await fire_failed_event(dispatcher) - scenario_result1 = await fire_failed_event(dispatcher) - scenario_result2 = make_scenario_result(scenario_result1.scenario) + event = event_cls(make_scenario_result()) - scenario_event = get_event(scenario_result2) - scheduler_.reset_mock() - sleep_.reset_mock() + with when, raises(BaseException) as exc_info: + await dispatcher.fire(event) + + with then: + assert exc_info.type is RepeaterExecutionInterrupted + assert str(exc_info.value) == "Stop repeating scenarios after the first failure" + + +@pytest.mark.usefixtures(repeater.__name__) +async def test_repeats_no_fail_fast(*, dispatcher: Dispatcher, scheduler_: Mock): + with given: + await fire_arg_parsed_event(dispatcher, repeats=2, fail_fast_on_repeat=True) + await fire_startup_event(dispatcher, scheduler_) + await fire_passed_event(dispatcher) + + scenario_result = make_scenario_result().mark_failed() + event = ScenarioRunEvent(scenario_result) with when: - await dispatcher.fire(scenario_event) + await dispatcher.fire(event) with then: - assert scheduler_.mock_calls == [call.schedule(scenario_result2.scenario)] - assert sleep_.mock_calls == [call(repeats_delay)] + # no exception + pass @pytest.mark.parametrize("repeats", [2, 3]) @@ -191,7 +239,7 @@ async def test_add_summary_with_delay(repeats: int, repeats_delay: int, *, @pytest.mark.usefixtures(repeater.__name__) -async def test_dont_add_summary(dispatcher: Dispatcher, scheduler_: Mock): +async def test_dont_add_summary_no_repeats(dispatcher: Dispatcher, scheduler_: Mock): with given: await fire_arg_parsed_event(dispatcher, repeats=1) await fire_startup_event(dispatcher, scheduler_) @@ -209,29 +257,19 @@ async def test_dont_add_summary(dispatcher: Dispatcher, scheduler_: Mock): @pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=0) - - with then: - assert exc_info.type is ValueError - assert str(exc_info.value) == "--repeats must be >= 1" - - -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_delay_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=2, repeats_delay=-0.001) +async def test_dont_add_summary_interrupted(dispatcher: Dispatcher, scheduler_: Mock): + with given: + await fire_arg_parsed_event(dispatcher, repeats=2) + await fire_startup_event(dispatcher, scheduler_) - with then: - assert exc_info.type is ValueError - assert str(exc_info.value) == "--repeats-delay must be >= 0.0" + await fire_failed_event(dispatcher) + report = Report() + report.set_interrupted(make_exc_info(Exception())) + cleanup_event = CleanupEvent(report) -@pytest.mark.usefixtures(repeater.__name__) -async def test_repeat_delay_without_repeats_validation(dispatcher: Dispatcher): - with when, raises(BaseException) as exc_info: - await fire_arg_parsed_event(dispatcher, repeats=1, repeats_delay=0.1) + with when: + await dispatcher.fire(cleanup_event) with then: - assert exc_info.type is ValueError + assert report.summary == []