From 74de31cb5ed815308354eefefda7539abf2dc020 Mon Sep 17 00:00:00 2001 From: Sandro Gehri Date: Mon, 4 Dec 2023 16:46:43 +0100 Subject: [PATCH] Add Retry support --- README.md | 17 ++++++ WIP.md | 5 +- docs/mutation-testing.md | 17 ++++++ src/Contracts/Configuration.php | 2 + src/Decorators/TestCallDecorator.php | 11 ++++ src/MutationTest.php | 13 +++++ src/MutationTestCollection.php | 34 ++++++++++++ src/Options/RetryOption.php | 28 ++++++++++ src/Repositories/ConfigurationRepository.php | 1 + src/Repositories/MutationRepository.php | 17 ++++++ .../Configuration/AbstractConfiguration.php | 16 +++++- .../Configuration/CliConfiguration.php | 7 +++ src/Support/Configuration/Configuration.php | 1 + src/Support/MutationTestResult.php | 12 ++-- src/Support/ResultCache.php | 55 +++++++++++++++++++ src/Tester/MutationTestRunner.php | 6 ++ ...HandleTestCallProfileConfigurationTest.php | 9 +++ .../HandlesCliProfileConfigurationTest.php | 12 ++++ .../HandlesGlobalProfileConfigurationTest.php | 15 +++++ 19 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 src/Options/RetryOption.php create mode 100644 src/Support/ResultCache.php diff --git a/README.md b/README.md index e270b69..f8c6e4e 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ The following options are available. - [`stopOnSurvived()`](#stopOnSurvived) - [`stopOnNotCovered()`](#stopOnNotCovered) - [`bail()`](#bail) +- [`retry()`](#retry) - [`min()`](#min) - [`ignoreMinScoreOnZeroMutations()`](#ignoreMinScoreOnZeroMutations) @@ -298,6 +299,22 @@ mutate() ``` + +### `retry()` +CLI: `--retry` + +If a mutation previously survived, you typically want to run them first. In such cases, you can use the `--retry` option. + +The `--retry` flag reorders your mutations by prioritizing the previously survived mutations. If there were no past survivals, the suite runs as usual. + +Additionally, it will stop execution upon first survived mutant. + +```php +mutate() + ->retry(); +``` + + ### `min()` CLI: `--min` diff --git a/WIP.md b/WIP.md index d90cb17..8caa088 100644 --- a/WIP.md +++ b/WIP.md @@ -19,7 +19,7 @@ - [ ] Allow registering Custom Mutators - [ ] Disable mutations by annotation - [x] Caching -- [ ] Order mutations to execute +- [x] Retry (runs survived mutations first) - [ ] Verbose output - [ ] Text log - [ ] HTML report @@ -29,8 +29,6 @@ # Known Bugs # Backlog Prio 1 -- [ ] Run test that killed a mutation first -- [ ] Run mutations in a reasonable order: New, Survived, NotCovered, Skipped, Killed (Survived first, if --stop-on-survived or --bail; NotCovered first, if --stop-on-uncovered) - [ ] Automatically empty cache when package version changes / Maybe there is another approach: Use the same cache key per php file, but store a hash of the file content and the package version in the cache. If the hash changes, the cache is invalid. - [ ] Automatically skip "Arch" and "Stressless" tests - wait for arch() and stress() aliases for test() - [ ] Properly support xdebug @@ -46,7 +44,6 @@ - [ ] Dedicated help output (`vendor/bin/pest --mutate --help`) - [ ] Add help to show available mutators and sets - [ ] Add more Laravel mutators -- [ ] Improve test filtering. Some test names, may not work - [ ] Better loop detection. For example when mutate break to continue in a while true loop - [ ] Beautify output - [ ] HTML report diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md index e06fd51..2fa4e4c 100644 --- a/docs/mutation-testing.md +++ b/docs/mutation-testing.md @@ -130,6 +130,7 @@ The following options are available. - [`stopOnSurvived()`](#stopOnSurvived) - [`stopOnNotCovered()`](#stopOnNotCovered) - [`bail()`](#bail) +- [`retry()`](#retry) - [`min()`](#min) - [`ignoreMinScoreOnZeroMutations()`](#ignoreMinScoreOnZeroMutations) @@ -288,6 +289,22 @@ mutate() ``` + +### `retry()` +CLI: `--retry` + +If a mutation previously survived, you typically want to run them first. In such cases, you can use the `--retry` option. + +The `--retry` flag reorders your mutations by prioritizing the previously survived mutations. If there were no past survivals, the suite runs as usual. + +Additionally, it will stop execution upon first survived mutant. + +```php +mutate() + ->retry(); +``` + + ### `min()` CLI: `--min` diff --git a/src/Contracts/Configuration.php b/src/Contracts/Configuration.php index 4593d3b..a230fa6 100644 --- a/src/Contracts/Configuration.php +++ b/src/Contracts/Configuration.php @@ -52,4 +52,6 @@ public function class(array|string ...$classes): self; public function uncommittedOnly(bool $uncommittedOnly = true): self; public function changedOnly(?string $branch = 'main'): self; + + public function retry(bool $retry = true): self; } diff --git a/src/Decorators/TestCallDecorator.php b/src/Decorators/TestCallDecorator.php index 745b07f..ba7078d 100644 --- a/src/Decorators/TestCallDecorator.php +++ b/src/Decorators/TestCallDecorator.php @@ -187,5 +187,16 @@ public function changedOnly(?string $branch = 'main'): Configuration return $this; } + + public function retry(bool $retry = true): Configuration + { + $this->configuration->retry($retry); + + if ($retry) { + $this->configuration->stopOnSurvived(); + } + + return $this; + } } // @codeCoverageIgnoreEnd diff --git a/src/MutationTest.php b/src/MutationTest.php index 6654079..1ac3f4f 100644 --- a/src/MutationTest.php +++ b/src/MutationTest.php @@ -28,6 +28,11 @@ public function __construct(public readonly Mutation $mutation) { } + public function getId(): string + { + return hash('xxh3', $this->mutation->file->getRealPath().$this->mutation->mutator.$this->mutation->startLine); + } + public function result(): MutationTestResult { return $this->result; @@ -154,4 +159,12 @@ public function duration(): float return $this->finish - $this->start; } + + /** + * @param array $results + */ + public function lastRunResult(array $results): MutationTestResult + { + return MutationTestResult::from($results[$this->getId()] ?? 'none'); + } } diff --git a/src/MutationTestCollection.php b/src/MutationTestCollection.php index 33b1a47..7ff9c97 100644 --- a/src/MutationTestCollection.php +++ b/src/MutationTestCollection.php @@ -5,6 +5,7 @@ namespace Pest\Mutate; use Pest\Mutate\Support\MutationTestResult; +use Pest\Mutate\Support\ResultCache; use Symfony\Component\Finder\SplFileInfo; class MutationTestCollection @@ -60,4 +61,37 @@ public function notRun(): int { return count(array_filter($this->tests, fn (MutationTest $test): bool => $test->result() === MutationTestResult::None)); } + + public function hasLastRunSurvivedMutation(): bool + { + return array_filter(ResultCache::instance()->get($this), fn (string $result): bool => $result === MutationTestResult::Survived->value) !== []; + } + + /** + * @return array + */ + public function results(): array + { + $results = []; + + foreach ($this->tests as $test) { + if ($test->result() !== MutationTestResult::None) { + $results[$test->getId()] = $test->result()->value; + } + } + + return $results; + } + + public function sortBySurvivedFirst(): void + { + $lastRunResults = ResultCache::instance()->get($this); + + usort($this->tests, fn (MutationTest $a, MutationTest $b): int => ($b->lastRunResult($lastRunResults) === MutationTestResult::Survived) <=> ($a->lastRunResult($lastRunResults) === MutationTestResult::Survived)); + } + + public function isComplete(): bool + { + return array_filter($this->tests, fn (MutationTest $test): bool => $test->result() === MutationTestResult::None) === []; + } } diff --git a/src/Options/RetryOption.php b/src/Options/RetryOption.php new file mode 100644 index 0000000..95a99b2 --- /dev/null +++ b/src/Options/RetryOption.php @@ -0,0 +1,28 @@ +tests, fn (MutationTestCollection $a, MutationTestCollection $b): int => $b->hasLastRunSurvivedMutation() <=> $a->hasLastRunSurvivedMutation()); + + foreach ($this->tests as $testCollection) { + $testCollection->sortBySurvivedFirst(); + } + } + + public function saveResults(): void + { + foreach ($this->tests as $testCollection) { + ResultCache::instance()->put($testCollection); + } + } } diff --git a/src/Support/Configuration/AbstractConfiguration.php b/src/Support/Configuration/AbstractConfiguration.php index 6bdd248..228fa04 100644 --- a/src/Support/Configuration/AbstractConfiguration.php +++ b/src/Support/Configuration/AbstractConfiguration.php @@ -56,6 +56,8 @@ abstract class AbstractConfiguration implements ConfigurationContract private ?string $changedOnly = null; + private ?bool $retry = null; + /** * {@inheritDoc} */ @@ -189,7 +191,7 @@ public function changedOnly(?string $branch = 'main'): self } /** - * @return array{paths?: string[], paths_to_ignore?: string[], mutators?: class-string[], excluded_mutators?: class-string[], classes?: string[], parallel?: bool, processes?: int, profile?: bool, min_score?: float, ignore_min_score_on_zero_mutations?: bool, covered_only?: bool, stop_on_survived?: bool, stop_on_not_covered?: bool, uncommitted_only?: bool, changed_only?: string} + * @return array{paths?: string[], paths_to_ignore?: string[], mutators?: class-string[], excluded_mutators?: class-string[], classes?: string[], parallel?: bool, processes?: int, profile?: bool, min_score?: float, ignore_min_score_on_zero_mutations?: bool, covered_only?: bool, stop_on_survived?: bool, stop_on_not_covered?: bool, uncommitted_only?: bool, changed_only?: string, retry?: bool} */ public function toArray(): array { @@ -209,6 +211,7 @@ public function toArray(): array 'stop_on_not_covered' => $this->stopOnNotCovered, 'uncommitted_only' => $this->uncommittedOnly, 'changed_only' => $this->changedOnly, + 'retry' => $this->retry, ], fn (mixed $value): bool => ! is_null($value)); } @@ -244,4 +247,15 @@ function (string $mutator): string { return $mutators; // @phpstan-ignore-line } + + public function retry(bool $retry = true): self + { + $this->retry = $retry; + + if ($retry) { + $this->stopOnSurvived = true; + } + + return $this; + } } diff --git a/src/Support/Configuration/CliConfiguration.php b/src/Support/Configuration/CliConfiguration.php index c52bf39..ba132c5 100644 --- a/src/Support/Configuration/CliConfiguration.php +++ b/src/Support/Configuration/CliConfiguration.php @@ -18,6 +18,7 @@ use Pest\Mutate\Options\PathOption; use Pest\Mutate\Options\ProcessesOption; use Pest\Mutate\Options\ProfileOption; +use Pest\Mutate\Options\RetryOption; use Pest\Mutate\Options\StopOnNotCoveredOption; use Pest\Mutate\Options\StopOnSurvivedOption; use Pest\Mutate\Options\UncommittedOnlyOption; @@ -44,6 +45,7 @@ class CliConfiguration extends AbstractConfiguration BailOption::class, UncommittedOnlyOption::class, ChangedOnlyOption::class, + RetryOption::class, ]; /** @@ -147,6 +149,11 @@ public function fromArguments(array $arguments): array $this->changedOnly($input->getOption(ChangedOnlyOption::ARGUMENT) !== null ? (string) $input->getOption(ChangedOnlyOption::ARGUMENT) : 'main'); // @phpstan-ignore-line } + if ($input->hasOption(RetryOption::ARGUMENT)) { + $this->retry($input->getOption(RetryOption::ARGUMENT) !== 'false'); + + } + return $arguments; } } diff --git a/src/Support/Configuration/Configuration.php b/src/Support/Configuration/Configuration.php index 1a72f9a..3e5e8ac 100644 --- a/src/Support/Configuration/Configuration.php +++ b/src/Support/Configuration/Configuration.php @@ -29,6 +29,7 @@ public function __construct( public readonly bool $stopOnNotCovered, public readonly bool $uncommittedOnly, public readonly string|false $changedOnly, + public readonly bool $retry, ) { } } diff --git a/src/Support/MutationTestResult.php b/src/Support/MutationTestResult.php index a3ca417..5638cf0 100644 --- a/src/Support/MutationTestResult.php +++ b/src/Support/MutationTestResult.php @@ -4,11 +4,11 @@ namespace Pest\Mutate\Support; -enum MutationTestResult +enum MutationTestResult: string { - case None; - case NotCovered; - case Killed; - case Survived; - case Timeout; + case None = 'none'; + case NotCovered = 'not-covered'; + case Killed = 'killed'; + case Survived = 'survived'; + case Timeout = 'timeout'; } diff --git a/src/Support/ResultCache.php b/src/Support/ResultCache.php new file mode 100644 index 0000000..6862552 --- /dev/null +++ b/src/Support/ResultCache.php @@ -0,0 +1,55 @@ +> + */ + private array $results = []; + + public static function instance(): self + { + return self::$instance ?? self::$instance = new self(); + } + + public function __construct() + { + $this->cache = new FileCache(); + } + + /** + * @return array + */ + public function get(MutationTestCollection $testCollection): array + { + return $this->results[$this->key($testCollection)] ?? // @phpstan-ignore-line + $this->results[$this->key($testCollection)] = $this->cache->get($this->key($testCollection), []); // @phpstan-ignore-line + } + + public function put(MutationTestCollection $testCollection): void + { + if ($testCollection->isComplete()) { + $this->cache->set($this->key($testCollection), $testCollection->results()); + + return; + } + + $this->cache->set($this->key($testCollection), [...$this->get($testCollection), ...$testCollection->results()]); + } + + private function key(MutationTestCollection $testCollection): string + { + return 'test-result-'.hash('xxh3', $testCollection->file->getRealPath()); + } +} diff --git a/src/Tester/MutationTestRunner.php b/src/Tester/MutationTestRunner.php index 461eff9..c5565d6 100644 --- a/src/Tester/MutationTestRunner.php +++ b/src/Tester/MutationTestRunner.php @@ -157,6 +157,10 @@ classesToMutate: $this->getConfiguration()->classes, Facade::instance()->emitter()->finishMutationGeneration($mutationSuite); + if ($this->getConfiguration()->retry) { + $mutationSuite->repository->sortBySurvivedFirst(); + } + Facade::instance()->emitter()->startMutationSuite($mutationSuite); if ($this->getConfiguration()->parallel) { @@ -172,6 +176,8 @@ classesToMutate: $this->getConfiguration()->classes, ); } + $mutationSuite->repository->saveResults(); + Facade::instance()->emitter()->finishMutationSuite($mutationSuite); $this->ensureMinScoreIsReached($mutationSuite); diff --git a/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php b/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php index a67ff29..c72290d 100644 --- a/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php +++ b/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php @@ -156,3 +156,12 @@ ->toBeTrue(); })->mutate(ConfigurationRepository::FAKE.'_17') ->profile(true); + +it('sets the retry option from test', function (): void { + $configuration = $this->repository->fakeTestConfiguration(ConfigurationRepository::FAKE.'_18'); + + expect($configuration->toArray()) + ->retry->toBeTrue() + ->stop_on_survived->toBeTrue(); +})->mutate(ConfigurationRepository::FAKE.'_18') + ->retry(true); diff --git a/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php b/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php index e909f76..cb16360 100644 --- a/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php +++ b/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php @@ -208,3 +208,15 @@ expect($this->configuration->toArray()) ->profile->toBeTrue(); }); + +it('enables profile option if --retry argument is passed', function (): void { + $this->configuration->fromArguments(['--mutate='.ConfigurationRepository::FAKE]); + expect($this->configuration->toArray()) + ->retry->toBeNull() + ->stop_on_survived->toBeNull(); + + $this->configuration->fromArguments(['--retry']); + expect($this->configuration->toArray()) + ->retry->toBeTrue() + ->stop_on_survived->toBeTrue(); +}); diff --git a/tests/Features/Profiles/HandlesGlobalProfileConfigurationTest.php b/tests/Features/Profiles/HandlesGlobalProfileConfigurationTest.php index 5bc78be..9ac3638 100644 --- a/tests/Features/Profiles/HandlesGlobalProfileConfigurationTest.php +++ b/tests/Features/Profiles/HandlesGlobalProfileConfigurationTest.php @@ -220,3 +220,18 @@ expect($this->configuration->toArray()['profile']) ->toBeFalse(); }); + +test('globally configure retry option', function (): void { + mutate(ConfigurationRepository::FAKE) + ->retry(); + + expect($this->configuration->toArray()) + ->retry->toBeTrue() + ->stop_on_survived->toBeTrue(); + + mutate(ConfigurationRepository::FAKE) + ->retry(false); + + expect($this->configuration->toArray()['retry']) + ->toBeFalse(); +});