Skip to content

Commit

Permalink
build: move benchmark execution to script and build native add-ons
Browse files Browse the repository at this point in the history
---
type: pre_commit_static_analysis_report
description: Results of running static analysis checks when committing changes.
report:
  - task: lint_filenames
    status: passed
  - task: lint_editorconfig
    status: passed
  - task: lint_markdown
    status: na
  - task: lint_package_json
    status: na
  - task: lint_repl_help
    status: na
  - task: lint_javascript_src
    status: na
  - task: lint_javascript_cli
    status: na
  - task: lint_javascript_examples
    status: na
  - task: lint_javascript_tests
    status: na
  - task: lint_javascript_benchmarks
    status: na
  - task: lint_python
    status: na
  - task: lint_r
    status: na
  - task: lint_c_src
    status: na
  - task: lint_c_examples
    status: na
  - task: lint_c_benchmarks
    status: na
  - task: lint_c_tests_fixtures
    status: na
  - task: lint_shell
    status: passed
  - task: lint_typescript_declarations
    status: na
  - task: lint_typescript_tests
    status: na
  - task: lint_license_headers
    status: passed
---

---
type: pre_push_report
description: Results of running various checks prior to pushing changes.
report:
  - task: run_javascript_examples
    status: na
  - task: run_c_examples
    status: na
  - task: run_cpp_examples
    status: na
  - task: run_javascript_readme_examples
    status: na
  - task: run_c_benchmarks
    status: na
  - task: run_cpp_benchmarks
    status: na
  - task: run_fortran_benchmarks
    status: na
  - task: run_javascript_benchmarks
    status: na
  - task: run_julia_benchmarks
    status: na
  - task: run_python_benchmarks
    status: na
  - task: run_r_benchmarks
    status: na
  - task: run_javascript_tests
    status: na
---
  • Loading branch information
Planeshifter committed Jan 20, 2025
1 parent 895313e commit 2eaf06c
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 19 deletions.
44 changes: 25 additions & 19 deletions .github/workflows/run_affected_benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ on:
# List paths for which changes should trigger this workflow:
- 'lib/**/benchmark/**'

workflow_dispatch:
inputs:
directories:
description: 'List of changed directories for which to run affected benchmarks (space-separated)'

# Global permissions:
permissions:
# Allow read-only access to the repository contents:
Expand Down Expand Up @@ -103,9 +108,10 @@ jobs:
make init
timeout-minutes: 5

# Get list of changed files:
- name: 'Get list of changed files'
id: changed-files
# Get list of changed directories from PR and push events:
- name: 'Get list of changed directories'
if: github.event_name != 'workflow_dispatch'
id: changed-directories
continue-on-error: true
run: |
if [ -n "${{ github.event.pull_request.number }}" ]; then
Expand All @@ -120,24 +126,24 @@ jobs:
files=$(git diff --diff-filter=AM --name-only ${{ github.event.before }} ${{ github.event.after }})
fi
fi
# Keep only benchmark files:
files=$(echo "$files" | grep -E 'benchmark/' | tr '\n' ' ' | sed 's/ $//')
echo "files=${files}" >> $GITHUB_OUTPUT
directories=$(for file in $files; do dirname $file; done | uniq | tr '\n' ' ' | sed 's/ $//')
echo "directories=${directories}" >> $GITHUB_OUTPUT
# Run JavaScript benchmarks:
- name: 'Run JavaScript benchmarks'
# Get list of changed directories from workflow dispatch event:
- name: 'Get list of changed directories (from user input)'
if: github.event_name == 'workflow_dispatch'
id: changed-directories-user-input
run: |
files=$(echo "${{ steps.changed-files.outputs.files }}" | tr ' ' '\n' | grep -E '\.js$' | tr '\n' ' ' | sed 's/ $//')
if [ -n "$files" ]; then
make benchmark-javascript-files FILES="${files}"
fi
timeout-minutes: 30
echo "directories=${{ github.event.inputs.directories }}" >> $GITHUB_OUTPUT
timeout-minutes: 5

# Run C benchmarks:
- name: 'Run C benchmarks'
# Run affected benchmarks:
- name: 'Run affected benchmarks'
run: |
files=$(echo "${{ steps.changed-files.outputs.files }}" | tr ' ' '\n' | grep -E '\.c$' | sed "s|^|${GITHUB_WORKSPACE}/|" | tr '\n' ' ' | sed 's/ $//')
if [ -n "$files" ]; then
make benchmark-c-files FILES="${files}"
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
directories="${{ steps.changed-directories-user-input.outputs.directories }}"
else
directories="${{ steps.changed-directories.outputs.directories }}"
fi
timeout-minutes: 15
. "$GITHUB_WORKSPACE/.github/workflows/scripts/run_affected_benchmarks" "$directories"
timeout-minutes: 30
163 changes: 163 additions & 0 deletions .github/workflows/scripts/run_affected_benchmarks
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
#!/usr/bin/env bash
#
# @license Apache-2.0
#
# Copyright (c) 2025 The Stdlib Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script to run affected benchmarks for a given list of changed paths.
#
# Usage: run_affected_benchmarks path1 [path2 path3 ...]
#
# Arguments:
#
# path1 File name or directory path.
# path2 File name or directory path.
# path3 File name or directory path.
#
#
# Environment variables:
#
# LOG_FILE Log file.
#

# shellcheck disable=SC2034,SC2153,SC2317

# Ensure that the exit status of pipelines is non-zero in the event that at least one of the commands in a pipeline fails:
set -o pipefail


# VARIABLES #

# Get the list of changed files:
changed="$*"

# Get the path to a log file as the third argument to the build script:
log_file="${LOG_FILE}"

# Define a heartbeat interval to periodically print messages in order to prevent CI from prematurely ending a build due to long-running commands:
heartbeat_interval='30s'

# Declare a variable for storing the heartbeat process id:
heartbeat_pid=""


# FUNCTIONS #

# Error handler.
#
# $1 - error status
on_error() {
echo 'ERROR: An error was encountered during execution.' >&2
cleanup
exit "$1"
}

# Runs clean-up tasks.
cleanup() {
stop_heartbeat
}

# Starts a heartbeat.
#
# $1 - heartbeat interval
start_heartbeat() {
echo 'Starting heartbeat...' >&2

# Create a heartbeat and send to background:
heartbeat "$1" &

# Capture the heartbeat pid:
heartbeat_pid=$!
echo "Heartbeat pid: ${heartbeat_pid}" >&2
}

# Runs an infinite print loop.
#
# $1 - heartbeat interval
heartbeat() {
while true; do
echo "$(date) - heartbeat..." >&2
sleep "$1"
done
}

# Stops the heartbeat print loop.
stop_heartbeat() {
echo 'Stopping heartbeat...' >&2
kill "${heartbeat_pid}"
}

# Prints a success message.
print_success() {
echo 'Success!' >&2
}

# Main execution sequence.
main() {
start_heartbeat "${heartbeat_interval}"

# Only keep files which reside in package directories:
changed=$(echo "${changed}" | tr ' ' '\n' | grep '^lib/node_modules/@stdlib') || true

# Find unique package directories:
directories=$(echo "${changed}" | tr ' ' '\n' | sed -E 's/\/(bin|data|etc|include|lib|src|test)\/?$//' | uniq)

if [ -z "${directories}" ]; then
echo 'No packages to run benchmarks for.' >&2
cleanup
print_success
exit 0
fi

# Extract package names from changed package directories (e.g., @stdlib/math/base/special/sin) by removing the leading 'lib/node_modules/':
packages=$(echo "${directories}" | sed -E 's/^lib\/node_modules\///')

# Build native add-ons for packages (if applicable):
for pkg in ${packages}; do
if [ -f "lib/node_modules/${pkg}/binding.gyp" ]; then
NODE_ADDONS_PATTERN="${pkg}" make install-node-addons
fi
done

# Find all benchmark files in package directories:
js_bench_files=$(find "${directories}" -maxdepth 3 -wholename '**/benchmark/benchmark*.js' | grep -v '/fixtures/' | sort -u | tr '\n' ' ') || true

This comment has been minimized.

Copy link
@kgryte

kgryte Jan 20, 2025

Member

@Planeshifter Here and below: can this handle benchmarks in nested folders? E.g., benchmark/c/native/... etc.


# Run JS benchmarks:
if [ -n "${js_bench_files}" ]; then
make benchmark-javascript-files FILES="${js_bench_files}"
else
echo 'No JavaScript benchmarks to run.' >&2
fi

# Run C benchmarks:
echo "Finding C benchmark files in ${directories}..."
c_bench_files=$(find "${directories}" -maxdepth 4 -wholename '**/benchmark/c/benchmark*.c' -exec realpath {} \; | grep -v '/fixtures/' | sort -u | tr '\n' ' ') || true

if [ -n "${c_bench_files}" ]; then
make benchmark-c-files FILES="${c_bench_files}"
else
echo 'No C benchmarks to run.' >&2
fi

cleanup
print_success
exit 0
}

# Set an error handler to print captured output and perform any clean-up tasks:
trap 'on_error' ERR

# Run main:
main

0 comments on commit 2eaf06c

Please sign in to comment.