From f7d96d95b633bb1dad39a7033e444f8f81907edd Mon Sep 17 00:00:00 2001 From: martinvuyk Date: Wed, 5 Mar 2025 11:38:00 -0300 Subject: [PATCH 1/2] Fix benchmarking for public repo Signed-off-by: martinvuyk --- mojo/stdlib/benchmarks/README.md | 16 +++------------- .../benchmarks/algorithm/bench_elementwise.mojo | 2 -- mojo/stdlib/benchmarks/builtin/bench_int.mojo | 2 -- mojo/stdlib/benchmarks/builtin/bench_sort.mojo | 2 -- .../benchmarks/collections/bench_dict.mojo | 2 -- .../benchmarks/collections/bench_string.mojo | 2 -- mojo/stdlib/benchmarks/hashlib/bench_hash.mojo | 2 -- mojo/stdlib/benchmarks/math/bench_math.mojo | 2 -- .../stdlib/benchmarks/utils/bench_formatter.mojo | 2 -- mojo/stdlib/benchmarks/utils/bench_memmem.mojo | 2 -- mojo/stdlib/scripts/run-benchmarks.sh | 15 +++++++++++++++ 11 files changed, 18 insertions(+), 31 deletions(-) diff --git a/mojo/stdlib/benchmarks/README.md b/mojo/stdlib/benchmarks/README.md index 54c94807f5..99fbb83e2c 100644 --- a/mojo/stdlib/benchmarks/README.md +++ b/mojo/stdlib/benchmarks/README.md @@ -17,19 +17,9 @@ internally. ## How to run the benchmarks -If you want to just compile and run all of the benchmarks as-is, -there is a script provided [here](../../stdlib/scripts/run-benchmarks.sh). -This script builds the open source `stdlib.mojopkg` and then executes -all of the benchmarks sequentially. The script also allows specifying a -subdirectory or a file to run. - -Running e.g. `magic run mojo run stdlib/benchmarks/collections/bench_dict.mojo` -makes the linker use the existing branch that the compiler is on. If you wish to -test changes you are making on the current branch, remove the `-t` flag on top -of the benchmark file (`# RUN: %mojo-no-debug %s -t`) then run: -`magic run stdlib/scripts/run-benchmarks.sh -stdlib/benchmarks/collections/bench_dict.mojo`. -Remember to replace the `-t` flag again before pushing any code. +Running e.g. `magic run benchmarks ./stdlib/benchmarks/collections` runs every +file in the directory sequentially using the build based on the current branch. +It can also run one file when specified. ## How to write effective benchmarks diff --git a/mojo/stdlib/benchmarks/algorithm/bench_elementwise.mojo b/mojo/stdlib/benchmarks/algorithm/bench_elementwise.mojo index 21d4cc80e2..b4dcf9b11e 100644 --- a/mojo/stdlib/benchmarks/algorithm/bench_elementwise.mojo +++ b/mojo/stdlib/benchmarks/algorithm/bench_elementwise.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from sys import simdwidthof diff --git a/mojo/stdlib/benchmarks/builtin/bench_int.mojo b/mojo/stdlib/benchmarks/builtin/bench_int.mojo index 8c25204216..03abac50cc 100644 --- a/mojo/stdlib/benchmarks/builtin/bench_int.mojo +++ b/mojo/stdlib/benchmarks/builtin/bench_int.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from benchmark import Bench, BenchConfig, Bencher, BenchId diff --git a/mojo/stdlib/benchmarks/builtin/bench_sort.mojo b/mojo/stdlib/benchmarks/builtin/bench_sort.mojo index d4a1e9180b..465a6d8b22 100644 --- a/mojo/stdlib/benchmarks/builtin/bench_sort.mojo +++ b/mojo/stdlib/benchmarks/builtin/bench_sort.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from random import * diff --git a/mojo/stdlib/benchmarks/collections/bench_dict.mojo b/mojo/stdlib/benchmarks/collections/bench_dict.mojo index fd23c64da8..de0e33957b 100644 --- a/mojo/stdlib/benchmarks/collections/bench_dict.mojo +++ b/mojo/stdlib/benchmarks/collections/bench_dict.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from collections import Dict, Optional from collections.dict import DictEntry diff --git a/mojo/stdlib/benchmarks/collections/bench_string.mojo b/mojo/stdlib/benchmarks/collections/bench_string.mojo index 8b3acdc9f0..cdb5947611 100644 --- a/mojo/stdlib/benchmarks/collections/bench_string.mojo +++ b/mojo/stdlib/benchmarks/collections/bench_string.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from collections import Dict, Optional from collections.string import String diff --git a/mojo/stdlib/benchmarks/hashlib/bench_hash.mojo b/mojo/stdlib/benchmarks/hashlib/bench_hash.mojo index 20a3b5c3c8..4bf283f373 100644 --- a/mojo/stdlib/benchmarks/hashlib/bench_hash.mojo +++ b/mojo/stdlib/benchmarks/hashlib/bench_hash.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from hashlib._ahash import ( MULTIPLE, diff --git a/mojo/stdlib/benchmarks/math/bench_math.mojo b/mojo/stdlib/benchmarks/math/bench_math.mojo index 8d10ed5f93..2491f8f691 100644 --- a/mojo/stdlib/benchmarks/math/bench_math.mojo +++ b/mojo/stdlib/benchmarks/math/bench_math.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from math import * from random import * diff --git a/mojo/stdlib/benchmarks/utils/bench_formatter.mojo b/mojo/stdlib/benchmarks/utils/bench_formatter.mojo index eb9dea9e46..665536211b 100644 --- a/mojo/stdlib/benchmarks/utils/bench_formatter.mojo +++ b/mojo/stdlib/benchmarks/utils/bench_formatter.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from collections.string.string_slice import _memchr, _memmem from sys import simdwidthof diff --git a/mojo/stdlib/benchmarks/utils/bench_memmem.mojo b/mojo/stdlib/benchmarks/utils/bench_memmem.mojo index e33d34f3e9..17f6c711a2 100644 --- a/mojo/stdlib/benchmarks/utils/bench_memmem.mojo +++ b/mojo/stdlib/benchmarks/utils/bench_memmem.mojo @@ -11,8 +11,6 @@ # limitations under the License. # ===----------------------------------------------------------------------=== # # RUN: %mojo-no-debug %s -t -# NOTE: to test changes on the current branch using run-benchmarks.sh, remove -# the -t flag. Remember to replace it again before pushing any code. from collections.string.string_slice import _align_down, _memchr, _memmem from sys import simdwidthof diff --git a/mojo/stdlib/scripts/run-benchmarks.sh b/mojo/stdlib/scripts/run-benchmarks.sh index a8c1b427ba..470a5b350d 100755 --- a/mojo/stdlib/scripts/run-benchmarks.sh +++ b/mojo/stdlib/scripts/run-benchmarks.sh @@ -31,5 +31,20 @@ if [[ $# -gt 0 ]]; then BENCHMARK_PATH=$1 fi +# HACK: this is an absolutely horrible hack, please fix this +if [[ -z "${_START_MODULAR_INCLUDED+x}" ]]; then + # save the original stdlib + mv -f ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg ${BUILD_DIR}/og_stdlib.mojopkg + # copy the newly built stdlib + cp -f ${BUILD_DIR}/stdlib.mojopkg ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg + # return the original stdlib + mv -f ${BUILD_DIR}/og_stdlib.mojopkg ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg +fi + # Run the benchmarks sequentially lit --succinct --show-all --workers 1 ${BENCHMARK_PATH} + +if [[ -z "${_START_MODULAR_INCLUDED+x}" ]]; then + # return the original stdlib + mv -f ${BUILD_DIR}/og_stdlib.mojopkg ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg +fi From cc8a6fbdfa5cf84bb9c7971d477dedef690b3099 Mon Sep 17 00:00:00 2001 From: martinvuyk Date: Sat, 8 Mar 2025 16:01:26 -0300 Subject: [PATCH 2/2] fix silly mistake Signed-off-by: martinvuyk --- mojo/stdlib/scripts/run-benchmarks.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/mojo/stdlib/scripts/run-benchmarks.sh b/mojo/stdlib/scripts/run-benchmarks.sh index 470a5b350d..b7d6296f45 100755 --- a/mojo/stdlib/scripts/run-benchmarks.sh +++ b/mojo/stdlib/scripts/run-benchmarks.sh @@ -37,8 +37,6 @@ if [[ -z "${_START_MODULAR_INCLUDED+x}" ]]; then mv -f ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg ${BUILD_DIR}/og_stdlib.mojopkg # copy the newly built stdlib cp -f ${BUILD_DIR}/stdlib.mojopkg ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg - # return the original stdlib - mv -f ${BUILD_DIR}/og_stdlib.mojopkg ${REPO_ROOT}/.magic/envs/default/lib/mojo/stdlib.mojopkg fi # Run the benchmarks sequentially