feat(plugin-lighthouse): add lighthouse runner (#549)

.github/workflows/ci.yml

@@ -105,6 +105,15 @@ jobs:
         uses: nrwl/nx-set-shas@v4
       - name: Install dependencies
         run: npm ci
+      - name: Set custom Chrome path for Windows only
+        if: matrix.os == 'windows-latest'
+        # This path is considered in `testing/setup/src/lib/chrome-path-setup.ts` and used in different test configurations
+        run: |
+          echo "CUSTOM_CHROME_PATH=C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+        shell: pwsh
+      #- name: Log all environment variables
+      #  run: |
+      #      printenv
       - name: Integration test affected projects
         run: npx nx affected -t integration-test --parallel=3 --coverage.enabled
 

CONTRIBUTING.md

@@ -12,6 +12,25 @@ Make sure to install dependencies:
 npm install
 ```
 
+## Environment Variables
+
+This table provides a quick overview of the environmental setup, with detailed explanations in the corresponding sections.
+
+| Feature                          | Local Default | CI Default         | Description                                                                                                                   |
+| -------------------------------- | ------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------- |
+| `env.INCLUDE_SLOW_TESTS` **❗️** | `false`       | `true`             | Controls inclusion of long-running tests. Overridden by setting. Details in the [Testing](#Testing) section.                  |
+| `env.CUSTOM_CHROME_PATH`         | N/A           | Windows **❗️❗️** | Path to Chrome executable. See [plugin-lighthouse/CONTRIBUTING.md](./packages/plugin-lighthouse/CONTRIBUTING.md#chrome-path). |
+| Quality Pipeline                 | Off           | On                 | Runs all plugins against the codebase.                                                                                        |
+
+**❗️** Test Inclusion Logic
+
+- `INCLUDE_SLOW_TESTS='false'` skips long tests.
+- without `INCLUDE_SLOW_TESTS`, tests run if `CI` is set.
+
+**❗️❗️** Windows specific path set only in CI
+
+- some setups also require this setting locally
+
 ## Development
 
 Refer to docs on [how to run tasks in Nx](https://nx.dev/core-features/run-tasks).
@@ -41,6 +60,15 @@ npx nx affected:lint
 npx nx code-pushup -- collect
 ```
 
+## Testing
+
+Some of the plugins have a longer runtime. In order to ensure better DX, longer tests are excluded by default when executing tests locally.
+
+You can control the execution of long-running tests over the `INCLUDE_SLOW_TESTS` environment variable.
+
+To change this setup, open (or create) the `.env` file in the root folder.
+Edit or add the environment variable there as follows: `INCLUDE_SLOW_TESTS=true`.
+
 ## Git
 
 Commit messages must follow [conventional commits](https://conventionalcommits.org/) format.

packages/plugin-lighthouse/CONTRIBUTING.md

@@ -0,0 +1,124 @@
+# Contributing
+
+## Setup
+
+Make sure to install dependencies:
+
+```sh
+npm install
+```
+
+### Chrome path
+
+In this plugin we provide Lighthouse functionality exposed over the `lighthousePlugin`.
+To test lighthouse properly we work with a predefined testing setup.
+
+On some OS there could be a problem finding the path to Chrome.
+
+We try to detect it automatically in the set-setup script.
+
+If no chrome path is detected the error looks like this: `Runtime error encountered: No Chrome installations found.`
+
+To prevent this from happening you have to provide the path manually in your `.env`:
+
+```bash
+CUSTOM_CHROME_PATH=/Applications/Google Chrome.app/Contents/MacOS/Google Chrome
+```
+
+In the CI you can set the env variable like this:
+
+```yml
+# ...
+- name: Set custom Chrome path for Windows only
+  if: matrix.os == 'windows-latest'
+  run: |
+    echo "CUSTOM_CHROME_PATH=C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+  shell: pwsh
+
+# Optional debug log
+- name: Log all environment variables
+  run: printenv
+# ...
+```
+
+We added consider this path in our `beforeAll` hook.
+
+```ts
+beforeEach(() => {
+  try {
+    vi.stubEnv('CHROME_PATH', getChromePath());
+  } catch (e) {
+    const customChromePath = process.env['CUSTOM_CHROME_PATH'];
+    if (customChromePath == null || customChromePath === '') {
+      throw new Error('Chrome path not found. Please read the in the packages CONTRIBUTING.md/#trouble-shooting section.');
+    }
+    vi.stubEnv('CHROME_PATH', customChromePath);
+  }
+});
+```
+
+### Testing chrome flags
+
+1. run `npx chrome-debug --<chromium-flag>` to pass terminal arguments to Chrome. E.g. `npx chrome-debug --headless=shell`.
+   `npx chrome-debug --headless=shell --@TODO-PUT-OTHER-EXAMPLE-FOR-FLAG`
+
+For a full list of available flags check out [this document](https://peter.sh/experiments/chromium-command-line-switches/).
+
+> [!NOTE]
+> To pass chrome flags to lighthouse you have to provide them under `--chrome-flags="<chrome-flags-as-array>"`.
+> E.g. `lighthouse https://example.com --chrome-flage="--headless=shell"`
+
+2. Check if the flag got accepted. This is quite unintuitive as we would expect the passed flag to be visible under `chrome://flags/` but as you can see in the screenshot it is not visible.
+   <img width="1202" alt="chrome-flags" src="./docs/images/chrome-flags.png">
+   Instead open `chrome://version/` and look under the "Command Line" section.  
+   <img width="1202" alt="chrome-chrome-version" src="./docs/images/chrome-version.png">
+
+### Chrome User Data
+
+To bootstrap Chrome with a predefined for setting we have to provide a couple of config files that we located under `<project-root>/mock/chromium-user-data`.
+When executing Lighthouse we provide the path to this folder over the `Flag` object.
+
+To generate initialise or edit the file structure under `chromium-user-data` do the following steps:
+
+1. Spin up Chrome by running `npx chrome-debug --user-data-dir=./packages/plugin-lighthouse/mock/chromium-user-data`
+   <img width="1202" alt="chrome-blank-screen" src="./docs/images/chrome-blank-screen.png">
+
+2. If you do this the first time you should already see content under `<project-root>/mock/chromium-user-data`
+3. Edit the configuration over the Chrome UI. E.g. adding a profile
+4. Close chromium and open it again, and you should see chromium bootstraps as the configured user
+   <img width="1202" alt="chrome-blank-screen-pre-configured" src="./docs/images/chrome-blank-screen-pre-configure.png">
+
+To reset the above just delete the folder and apply the settings again.
+
+_A helpful chromium setup is preconfigured with the following settings:_
+
+- A user profile is set up. This enables certain debugging related options as well as help to visually distinguish between test setups as the header bar is colored.
+  <img width="1202" alt="chrome-settings-manage-profile" src="./docs/images/chrome-settings-manage-profile.png">
+
+#### Resources
+
+- [chromium flags guide](https://www.chromium.org/developers/how-tos/run-chromium-with-flags/)
+
+## Troubleshooting
+
+1. Verify Chrome Installation  
+   Ensure Chrome is correctly installed and accessible to the Lighthouse process.
+   Run `npx chrome-debug` to test it. Read further under [chrome-path](#chrome-path)
+
+2. Increase Timeout  
+   Lighthouse has a longer runtime which can time out in different environments.
+   **Try increasing the test timeout** in `lighthouse-plugin.integration.test.ts` for `runner creation and execution` test suite.
+
+3. Turn on debug mode  
+   Show debug logs of Lighthouse. Set the following environment variable: `DEBUG='*'`
+
+4. Understand error messages (⏳ could also be because of timeout problems :D )
+
+- Could not find `report.json` (⏳)
+  ![lighthouse-error-2.png](./docs/images/lighthouse-error-2.png)
+- Lighthouse Error - `Could Not Connect to Chrome` (⏳)
+  ![lighthouse-error-1.png](./docs/images/lighthouse-error-1.png)
+  Your Chrome path is set incorrectly. Read further under [chrome-path](#chrome-path)
+- Lighthouse Error - `start lh:<any>:<performancemark>" performance mark has not been set` (⏳)  
+  ![lighthouse-error-3.png](./docs/images/lighthouse-error-3.png)
+  If this error pops up you are able to launch Chrome but had problems to communicate over the ports.

packages/plugin-lighthouse/README.md

@@ -1,3 +1,121 @@
 # @code-pushup/lighthouse-plugin
 
-TODO: docs
+[![npm](https://img.shields.io/npm/v/%40code-pushup%2Flighthouse-plugin.svg)](https://www.npmjs.com/package/@code-pushup/lighthouse-plugin)
+[![downloads](https://img.shields.io/npm/dm/%40code-pushup%2Flighthouse-plugin)](https://npmtrends.com/@code-pushup/lighthouse-plugin)
+[![dependencies](https://img.shields.io/librariesio/release/npm/%40code-pushup/lighthouse-plugin)](https://www.npmjs.com/package/@code-pushup/lighthouse-plugin?activeTab=dependencies)
+
+🕵️ **Code PushUp plugin for measuring web performance and quality with Lighthouse.** 🔥
+
+---
+
+The plugin parses your Lighthouse configuration and lints all audits of the official [Lighthouse](https://github.com/GoogleChrome/lighthouse/blob/main/readme.md#lighthouse-------) CLI.
+
+Detected Lighthouse audits are mapped to Code PushUp audits. Audit reports are calculated based on the [original implementation](https://googlechrome.github.io/lighthouse/scorecalc/).
+Additionally, Lighthouse categories are mapped to Code PushUp groups which can make it easier to assemble the categories.
+
+For more infos visit the [official docs](https://developer.chrome.com/docs/lighthouse/overview).
+
+## Getting started
+
+1. If you haven't already, install [@code-pushup/cli](../cli/README.md) and create a configuration file.
+
+2. Install as a dev dependency with your package manager:
+
+   ```sh
+   npm install --save-dev @code-pushup/lighthouse-plugin
+   ```
+
+   ```sh
+   yarn add --dev @code-pushup/lighthouse-plugin
+   ```
+
+   ```sh
+   pnpm add --save-dev @code-pushup/lighthouse-plugin
+   ```
+
+3. Add this plugin to the `plugins` array in your Code PushUp CLI config file (e.g. `code-pushup.config.ts`).
+
+   Pass in the URL you want to measure, along with optional [flags](#flags) and [config](#config) data.
+
+   ```ts
+   import lighthousePlugin from '@code-pushup/lighthouse-plugin';
+
+   export default {
+     // ...
+     plugins: [
+       // ...
+       await lighthousePlugin('https://example.com'),
+     ],
+   };
+   ```
+
+4. Run the CLI with `npx code-pushup collect` and view or upload the report (refer to [CLI docs](../cli/README.md)).
+
+### Optionally set up categories
+
+@TODO
+
+## Flags
+
+The plugin accepts a second optional argument, `flags`.
+
+`flags` is the Lighthouse [CLI flags](https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/cli/cli-flags.js#L80) as a JS object.
+
+Within the flags object a couple of other external configuration files can be referenced. E.g. `configPath` , `preset` or `budgetPath` reference external `json` or JavaScript files.
+
+For a complete list the [official documentation of CLI flags](https://github.com/GoogleChrome/lighthouse/blob/main/readme.md#cli-options)
+
+> [!TIP]  
+> If you are not used to work with the Lighthouse CLI you would pass flags like this:
+> `lighthouse https://example.com --output=json --chromeFlags='--headless=shell'`
+>
+> Now with the plugin it would look like this:
+>
+> ```ts
+> // code-pushup.config.ts
+> ...
+> lighthousePlugin('https://example.com', { output: 'json', chromeFlags: ['--headless=shell']});
+> ```
+
+## Config
+
+The plugin accepts a third optional argument, `config`.
+
+`config` is the Lighthouse [configuration](https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/types/config.d.ts#L21) as a JS object.
+
+For a complete guide on Lighthouse configuration read the [official documentation on configuring](https://github.com/GoogleChrome/lighthouse/blob/main/docs/configuration.md)
+
+> [!TIP]  
+> If you are not used to work with the Lighthouse CLI you would pass a config like this:
+> `lighthouse --config-path=path/to/custom-config.js https://example.com`
+>
+> And in a separate file you would place the following object:
+>
+> ```typescript
+> // custom-config.js file
+> export default {
+>   extends: 'lighthouse:default',
+>   settings: {
+>     onlyAudits: ['first-meaningful-paint', 'speed-index', 'interactive'],
+>   },
+> };
+> ```
+>
+> Now with the plugin it would look like this:
+>
+> ```ts
+> // code-pushup.config.ts
+> ...
+> lighthousePlugin('https://example.com', undefined, {
+>   extends: 'lighthouse:default',
+>   settings: {
+>     onlyAudits: [
+>       'first-meaningful-paint',
+>       'speed-index',
+>       'interactive',
+>      ],
+>  }
+> })
+> ```
+
+If you want to contribute, please refer to [CONTRIBUTING.md](./CONTRIBUTING.md).

packages/plugin-lighthouse/package.json

@@ -5,6 +5,8 @@
   "dependencies": {
     "@code-pushup/models": "*",
     "lighthouse": "^11.0.0",
-    "@code-pushup/utils": "*"
+    "@code-pushup/utils": "*",
+    "lighthouse-logger": "2.0.1",
+    "chalk": "^5.3.0"
   }
 }

packages/plugin-lighthouse/src/index.ts

@@ -1,3 +1,11 @@
 import { lighthousePlugin } from './lib/lighthouse-plugin';
 
+export { lighthousePlugin, LighthouseCliFlags } from './lib/lighthouse-plugin';
+export {
+  LIGHTHOUSE_REPORT_NAME,
+  LIGHTHOUSE_PLUGIN_SLUG,
+  LIGHTHOUSE_AUDITS,
+  LIGHTHOUSE_GROUPS,
+} from './lib/constants';
+
 export default lighthousePlugin;

packages/plugin-lighthouse/src/lib/constants.ts

@@ -1,4 +1,5 @@
 import {
+  type CliFlags,
   type Config,
   type IcuMessage,
   Audit as LHAudit,
@@ -11,7 +12,7 @@ export const LIGHTHOUSE_REPORT_NAME = 'lighthouse-report.json';
 
 const { audits, categories } = defaultConfig;
 
-export const GROUPS: Group[] = Object.entries(categories ?? {}).map(
+export const LIGHTHOUSE_GROUPS: Group[] = Object.entries(categories ?? {}).map(
   ([id, category]) => ({
     slug: id,
     title: getMetaString(category.title),
@@ -22,7 +23,7 @@ export const GROUPS: Group[] = Object.entries(categories ?? {}).map(
   }),
 );
 
-export const AUDITS: Audit[] = await Promise.all(
+export const LIGHTHOUSE_AUDITS: Audit[] = await Promise.all(
   (audits ?? []).map(async value => {
     const audit = await loadLighthouseAudit(value);
     return {
@@ -63,3 +64,21 @@ async function loadLighthouseAudit(
   };
   return module.default;
 }
+
+export const DEFAULT_CLI_FLAGS: Partial<CliFlags> = {
+  // default values extracted from
+  // https://github.com/GoogleChrome/lighthouse/blob/7d80178c37a1b600ea8f092fc0b098029799a659/cli/cli-flags.js#L80
+  verbose: false,
+  quiet: false,
+  saveAssets: false,
+  // needed to pass CI on linux and windows (locally it works without headless too)
+  chromeFlags: '--headless=shell',
+  port: 0,
+  hostname: '127.0.0.1',
+  view: false,
+  channel: 'cli',
+  chromeIgnoreDefaultFlags: false,
+  // custom overwrites in favour of the plugin
+  output: ['json'],
+  outputPath: LIGHTHOUSE_REPORT_NAME,
+};