Add Profiling support

pestphp · Dec 4, 2023 · 4c43b04 · 4c43b04
1 parent 95322ce
commit 4c43b04
Show file tree

Hide file tree

Showing 16 changed files with 212 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -403,6 +403,15 @@ Against a single mutation the tests are not run in parallel, regardless of the p
 vendor/bin/pest --mutate --parallel
 ```
 
+### Profiling
+
+You can profile the performance of the mutations by using the `--profile` option.
+It outputs a list of the then slowest mutations.
+
+```bash
+vendor/bin/pest --mutate --profile
+```
+
 ## Ignoring Mutations
 
 ### Ignore for a single line

diff --git a/WIP.md b/WIP.md
@@ -29,7 +29,7 @@
 # Known Bugs
 
 # Backlog Prio 1
-- [ ] Run test that killed a mutation before first
+- [ ] Run test that killed a mutation first
 - [ ] Run mutations in a reasonable order: New, Survived, NotCovered, Skipped, Killed (Survived first, if --stop-on-survived or --bail; NotCovered first, if --stop-on-uncovered)
 - [ ] Automatically empty cache when package version changes / Maybe there is another approach: Use the same cache key per php file, but store a hash of the file content and the package version in the cache. If the hash changes, the cache is invalid.
 - [ ] Automatically skip "Arch" and "Stressless" tests - wait for arch() and stress() aliases for test()

diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md
@@ -393,6 +393,15 @@ Against a single mutation the tests are not run in parallel, regardless of the p
 vendor/bin/pest --mutate --parallel
 ```
 
+### Profiling
+
+You can profile the performance of the mutations by using the `--profile` option.
+It outputs a list of the then slowest mutations.
+
+```bash
+vendor/bin/pest --mutate --profile
+```
+
 ## Ignoring Mutations
 
 ### Ignore for a single line

diff --git a/src/Contracts/Configuration.php b/src/Contracts/Configuration.php
@@ -36,6 +36,8 @@ public function parallel(bool $parallel = true): self;
 
     public function processes(int $processes = null): self;
 
+    public function profile(bool $profile = true): self;
+
     public function stopOnSurvived(bool $stopOnSurvived = true): self;
 
     public function stopOnNotCovered(bool $stopOnNotCovered = true): self;

diff --git a/src/Decorators/TestCallDecorator.php b/src/Decorators/TestCallDecorator.php
@@ -136,6 +136,13 @@ public function processes(int $processes = null): Configuration
         return $this;
     }
 
+    public function profile(bool $parallel = true): self
+    {
+        $this->configuration->profile($parallel);
+
+        return $this;
+    }
+
     /**
      * {@inheritDoc}
      */

diff --git a/src/MutationTest.php b/src/MutationTest.php
@@ -18,6 +18,10 @@ class MutationTest
 {
     private MutationTestResult $result = MutationTestResult::None;
 
+    private ?float $start = null;
+
+    private ?float $finish = null;
+
     private Process $process;
 
     public function __construct(public readonly Mutation $mutation)
@@ -85,6 +89,8 @@ public function start(array $coveredLines, Configuration $configuration, array $
             timeout: $this->calculateTimeout(),
         );
 
+        $this->start = microtime(true);
+
         $process->start();
 
         $this->process = $process;
@@ -113,6 +119,8 @@ public function hasFinished(): bool
 
             Facade::instance()->emitter()->mutationTimedOut($this);
 
+            $this->finish = microtime(true);
+
             return true;
         }
 
@@ -121,13 +129,29 @@ public function hasFinished(): bool
 
             Facade::instance()->emitter()->mutationSurvived($this);
 
+            $this->finish = microtime(true);
+
             return true;
         }
 
         $this->updateResult(MutationTestResult::Killed);
 
         Facade::instance()->emitter()->mutationKilled($this);
 
+        $this->finish = microtime(true);
+
         return true;
     }
+
+    public function duration(): float
+    {
+        if ($this->start === null) {
+            return 0;
+        }
+        if ($this->finish === null) {
+            return 0;
+        }
+
+        return $this->finish - $this->start;
+    }
 }
diff --git a/src/Options/ProfileOption.php b/src/Options/ProfileOption.php
@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Pest\Mutate\Options;
+
+use Symfony\Component\Console\Input\InputOption;
+
+class ProfileOption
+{
+    final public const ARGUMENT = 'profile';
+
+    public static function remove(): bool
+    {
+        return true;
+    }
+
+    public static function match(string $argument): bool
+    {
+        return $argument === sprintf('--%s', self::ARGUMENT) ||
+            str_starts_with($argument, sprintf('--%s=', self::ARGUMENT));
+    }
+
+    public static function inputOption(): InputOption
+    {
+        return new InputOption(sprintf('--%s', self::ARGUMENT), null, InputOption::VALUE_OPTIONAL, '');
+    }
+}
diff --git a/src/Repositories/ConfigurationRepository.php b/src/Repositories/ConfigurationRepository.php
@@ -101,6 +101,7 @@ public function mergedConfiguration(): Configuration
             classes: $config['classes'] ?? [],
             parallel: $parallel,
             processes: $parallel ? ($config['processes'] ?? (new CpuCoreCounter())->getCount()) : 1,
+            profile: $config['profile'] ?? false,
             minScore: $config['min_score'] ?? null,
             ignoreMinScoreOnZeroMutations: $config['ignore_min_score_on_zero_mutations'] ?? false,
             stopOnSurvived: $config['stop_on_survived'] ?? false,

diff --git a/src/Repositories/MutationRepository.php b/src/Repositories/MutationRepository.php
@@ -78,4 +78,18 @@ public function score(): float
 
         return ($this->killed() + $this->timedOut()) / $this->total() * 100;
     }
+
+    /**
+     * @return array<int, MutationTest>
+     */
+    public function slowest(): array
+    {
+        $allTests = array_merge(...array_values(array_map(fn (MutationTestCollection $testCollection): array => $testCollection->tests(), $this->tests)));
+
+        $allTests = array_filter($allTests, fn (MutationTest $test): bool => $test->duration() > 0);
+
+        usort($allTests, fn (MutationTest $a, MutationTest $b): int => $b->duration() <=> $a->duration());
+
+        return array_slice($allTests, 0, 10);
+    }
 }
diff --git a/src/Support/Configuration/AbstractConfiguration.php b/src/Support/Configuration/AbstractConfiguration.php
@@ -46,6 +46,8 @@ abstract class AbstractConfiguration implements ConfigurationContract
 
     private ?int $processes = null;
 
+    private ?bool $profile = null;
+
     private ?bool $stopOnSurvived = null;
 
     private ?bool $stopOnNotCovered = null;
@@ -133,6 +135,13 @@ public function processes(int $processes = null): self
         return $this;
     }
 
+    public function profile(bool $profile = true): self
+    {
+        $this->profile = $profile;
+
+        return $this;
+    }
+
     public function stopOnSurvived(bool $stopOnSurvived = true): self
     {
         $this->stopOnSurvived = $stopOnSurvived;
@@ -180,7 +189,7 @@ public function changedOnly(?string $branch = 'main'): self
     }
 
     /**
-     * @return array{paths?: string[], paths_to_ignore?: string[], mutators?: class-string<Mutator>[], excluded_mutators?: class-string<Mutator>[], classes?: string[], parallel?: bool, processes?: int, min_score?: float, ignore_min_score_on_zero_mutations?: bool, covered_only?: bool, stop_on_survived?: bool, stop_on_not_covered?: bool, uncommitted_only?: bool, changed_only?: string}
+     * @return array{paths?: string[], paths_to_ignore?: string[], mutators?: class-string<Mutator>[], excluded_mutators?: class-string<Mutator>[], classes?: string[], parallel?: bool, processes?: int, profile?: bool, min_score?: float, ignore_min_score_on_zero_mutations?: bool, covered_only?: bool, stop_on_survived?: bool, stop_on_not_covered?: bool, uncommitted_only?: bool, changed_only?: string}
      */
     public function toArray(): array
     {
@@ -192,6 +201,7 @@ public function toArray(): array
             'classes' => $this->classes,
             'parallel' => $this->parallel,
             'processes' => $this->processes,
+            'profile' => $this->profile,
             'min_score' => $this->minScore,
             'ignore_min_score_on_zero_mutations' => $this->ignoreMinScoreOnZeroMutations,
             'covered_only' => $this->coveredOnly,

diff --git a/src/Support/Configuration/CliConfiguration.php b/src/Support/Configuration/CliConfiguration.php
@@ -17,6 +17,7 @@
 use Pest\Mutate\Options\ParallelOption;
 use Pest\Mutate\Options\PathOption;
 use Pest\Mutate\Options\ProcessesOption;
+use Pest\Mutate\Options\ProfileOption;
 use Pest\Mutate\Options\StopOnNotCoveredOption;
 use Pest\Mutate\Options\StopOnSurvivedOption;
 use Pest\Mutate\Options\UncommittedOnlyOption;
@@ -37,6 +38,7 @@ class CliConfiguration extends AbstractConfiguration
         IgnoreOption::class,
         ParallelOption::class,
         ProcessesOption::class,
+        ProfileOption::class,
         StopOnSurvivedOption::class,
         StopOnNotCoveredOption::class,
         BailOption::class,
@@ -111,6 +113,15 @@ public function fromArguments(array $arguments): array
             $this->processes($input->getOption(ProcessesOption::ARGUMENT) !== null ? (int) $input->getOption(ProcessesOption::ARGUMENT) : null); // @phpstan-ignore-line
         }
 
+        if ($input->hasOption(ProfileOption::ARGUMENT)) {
+            $this->profile($input->getOption(ProfileOption::ARGUMENT) !== 'false');
+        }
+
+        if ($_SERVER['COLLISION_PRINTER_PROFILE'] ?? false) {
+            $this->profile(true);
+            unset($_SERVER['COLLISION_PRINTER_PROFILE']);
+        }
+
         if ($input->hasOption(ClassOption::ARGUMENT)) {
             $this->class(explode(',', (string) $input->getOption(ClassOption::ARGUMENT))); // @phpstan-ignore-line
         }

diff --git a/src/Support/Configuration/Configuration.php b/src/Support/Configuration/Configuration.php
@@ -22,6 +22,7 @@ public function __construct(
         public readonly array $classes,
         public readonly bool $parallel,
         public readonly int $processes,
+        public readonly bool $profile,
         public readonly ?float $minScore,
         public readonly bool $ignoreMinScoreOnZeroMutations,
         public readonly bool $stopOnSurvived,

diff --git a/src/Support/Printers/DefaultPrinter.php b/src/Support/Printers/DefaultPrinter.php
@@ -9,6 +9,7 @@
 use Pest\Mutate\MutationTest;
 use Pest\Mutate\MutationTestCollection;
 use Pest\Mutate\Repositories\ConfigurationRepository;
+use Pest\Mutate\Support\Configuration\Configuration;
 use Pest\Mutate\Support\MutationTestResult;
 use Pest\Support\Container;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -131,6 +132,9 @@ public function reportMutationSuiteStarted(MutationSuite $mutationSuite): void
 
     public function reportMutationSuiteFinished(MutationSuite $mutationSuite): void
     {
+        /** @var Configuration $configuration */
+        $configuration = Container::getInstance()->get(ConfigurationRepository::class)->mergedConfiguration(); // @phpstan-ignore-line
+
         if ($this->compact) {
             $this->output->writeln(''); // add new line after compact test output
         }
@@ -149,8 +153,8 @@ public function reportMutationSuiteFinished(MutationSuite $mutationSuite): void
         $duration = number_format($mutationSuite->duration(), 2);
         $this->output->writeln('  <fg=gray>Duration:</>  <fg=default>'.$duration.'s</>');
 
-        if (Container::getInstance()->get(ConfigurationRepository::class)->mergedConfiguration()->parallel) { // @phpstan-ignore-line
-            $processes = Container::getInstance()->get(ConfigurationRepository::class)->mergedConfiguration()->processes; // @phpstan-ignore-line
+        if ($configuration->parallel) {
+            $processes = $configuration->processes;
             $this->output->writeln('  <fg=gray>Parallel:</>  <fg=default>'.$processes.' processes</>');
         }
 
@@ -165,6 +169,10 @@ public function reportMutationSuiteFinished(MutationSuite $mutationSuite): void
         }
 
         $this->output->writeln('');
+
+        if ($configuration->profile) {
+            $this->writeMutationTestProfile($mutationSuite);
+        }
     }
 
     private function writeMutationTestLine(string $color, string $symbol, MutationTest $test): void
@@ -213,12 +221,58 @@ private function writeMutationTestSummary(MutationTest $test): void
 
         $diff = $test->mutation->diff;
         $this->output->writeln($diff);
+    }
+
+    private function writeMutationTestProfile(MutationSuite $mutationSuite): void
+    {
+        /** @var Configuration $configuration */
+        $configuration = Container::getInstance()->get(ConfigurationRepository::class)->mergedConfiguration(); // @phpstan-ignore-line
+
+        $this->output->writeln('  <fg=gray>Top 10 slowest tests:</>');
+
+        $timeElapsed = $mutationSuite->duration() * ($configuration->parallel ? $configuration->processes : 1);
+
+        $slowTests = $mutationSuite->repository->slowest();
+
+        foreach ($slowTests as $slowTest) {
+            $path = str_ireplace(getcwd().'/', '', (string) $slowTest->mutation->file->getRealPath());
+
+            $seconds = number_format($slowTest->duration(), 2, '.', '');
+
+            $color = ($slowTest->duration()) > $timeElapsed * 0.25 ? 'red' : ($slowTest->duration() > $timeElapsed * 0.1 ? 'yellow' : 'gray');
+
+            render(sprintf(<<<'HTML'
+                <div class="flex justify-between space-x-1 mx-2">
+                    <span class="flex-1">
+                        <span class="font-bold">%s</span><span class="text-gray mx-1">></span><span class="text-gray">Line %s: %s</span>
+                    </span>
+                    <span class="ml-1 font-bold text-%s">
+                        %ss
+                    </span>
+                </div>
+            HTML, $path, $slowTest->mutation->startLine, $slowTest->mutation->mutator::name(), $color, $seconds));
+        }
 
-        //        render(<<<HTML
-        //                        <div class="mx-2 flex">
-        //                            {$diff}
-        //                        </div>
-        //                    HTML
-        //        );
+        $timeElapsedInSlowTests = array_sum(array_map(fn (MutationTest $testResult): float => $testResult->duration(), $slowTests));
+
+        $timeElapsedAsString = number_format($timeElapsed, 2, '.', '');
+        $percentageInSlowTestsAsString = number_format($timeElapsedInSlowTests * 100 / $timeElapsed, 2, '.', '');
+        $timeElapsedInSlowTestsAsString = number_format($timeElapsedInSlowTests, 2, '.', '');
+
+        render(sprintf(<<<'HTML'
+            <div class="mx-2 mb-1 flex">
+                <div class="text-gray">
+                    <hr/>
+                </div>
+                <div class="flex space-x-1 justify-between">
+                    <span>
+                    </span>
+                    <span>
+                        <span class="text-gray">(%s%% of %ss)</span>
+                        <span class="ml-1 font-bold">%ss</span>
+                    </span>
+                </div>
+            </div>
+        HTML, $percentageInSlowTestsAsString, $timeElapsedAsString, $timeElapsedInSlowTestsAsString));
     }
 }
diff --git a/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php b/tests/Features/Profiles/HandleTestCallProfileConfigurationTest.php
@@ -148,3 +148,11 @@
         ->toBe('other-branch');
 })->mutate(ConfigurationRepository::FAKE.'_12')
     ->changedOnly('other-branch');
+
+it('sets the profile option from test', function (): void {
+    $configuration = $this->repository->fakeTestConfiguration(ConfigurationRepository::FAKE.'_17');
+
+    expect($configuration->toArray()['profile'])
+        ->toBeTrue();
+})->mutate(ConfigurationRepository::FAKE.'_17')
+    ->profile(true);
diff --git a/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php b/tests/Features/Profiles/HandlesCliProfileConfigurationTest.php
@@ -198,3 +198,13 @@
     expect($this->configuration->toArray())
         ->changed_only->toBe('other-branch');
 });
+
+it('enables profile option if --profile argument is passed', function (): void {
+    $this->configuration->fromArguments(['--mutate='.ConfigurationRepository::FAKE]);
+    expect($this->configuration->toArray())
+        ->profile->toBeNull();
+
+    $this->configuration->fromArguments(['--profile']);
+    expect($this->configuration->toArray())
+        ->profile->toBeTrue();
+});