From a352007c0e97c0ec373bbb93b8554b40f4eb1529 Mon Sep 17 00:00:00 2001 From: Alexis Montoison Date: Tue, 15 Feb 2022 18:20:47 -0500 Subject: [PATCH] Update benchmarks --- Jenkinsfile => benchmark/Jenkinsfile | 25 +++-- benchmark/Project.toml | 2 + benchmark/cg_bmark.jl | 36 +++---- benchmark/push_benchmarks.sh | 28 ------ benchmark/run_benchmarks.jl | 85 ----------------- benchmark/send_comment_to_pr.jl | 137 --------------------------- 6 files changed, 40 insertions(+), 273 deletions(-) rename Jenkinsfile => benchmark/Jenkinsfile (79%) delete mode 100644 benchmark/push_benchmarks.sh delete mode 100644 benchmark/run_benchmarks.jl delete mode 100644 benchmark/send_comment_to_pr.jl diff --git a/Jenkinsfile b/benchmark/Jenkinsfile similarity index 79% rename from Jenkinsfile rename to benchmark/Jenkinsfile index 43afeb3da..c62061240 100644 --- a/Jenkinsfile +++ b/benchmark/Jenkinsfile @@ -1,4 +1,4 @@ -def bmarkFile = 'run_benchmarks.jl' +def bmarkFile = 'benchmarks.jl' def prNumber = BRANCH_NAME.tokenize("PR-")[0] pipeline { agent any @@ -87,11 +87,25 @@ pipeline { git checkout $LOCAL_BRANCH_NAME -- ''' } + sh "git clone https://github.com/ProofOfConceptForJuliSmoothOptimizers/BenchmarkSetup.git || true" + } + } + stage('setup benchmark repository') { + when { + expression { env.comment } + } + steps { + dir(WORKSPACE + "/BenchmarkSetup") { + sh ''' + git fetch origin + git pull --ff-only origin master + ''' + } } } stage('run benchmarks') { when { - expression {env.comment} + expression { env.comment } } steps { script { @@ -101,9 +115,9 @@ pipeline { } } dir(WORKSPACE + "/$repo") { - sh "mkdir -p $HOME/benchmarks/${org}/${repo}" - sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log benchmark/push_benchmarks.sh $bmarkFile" - } + sh "mkdir -p $HOME/benchmarks/${org}/${repo}" + sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log ../BenchmarkSetup/benchmark/push_benchmarks.sh ${bmarkFile}" + } } } } @@ -116,4 +130,3 @@ pipeline { } } } - diff --git a/benchmark/Project.toml b/benchmark/Project.toml index 417790eee..2a34b11a1 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -13,8 +13,10 @@ KrylovMethods = "9a2cd570-f05c-5dc1-9209-93ad6f5727f7" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MatrixMarket = "4d4711f2-db25-561a-b6b3-d35e7d4047d3" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +PlotThemes = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" SolverBenchmark = "581a75fa-a23a-52d0-a590-d6201de2218a" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SuiteSparseMatrixCollection = "ac199af8-68bc-55b8-82c4-7abd6f96ed98" diff --git a/benchmark/cg_bmark.jl b/benchmark/cg_bmark.jl index c1d1d7b3c..af9c84ad2 100644 --- a/benchmark/cg_bmark.jl +++ b/benchmark/cg_bmark.jl @@ -14,39 +14,41 @@ using SuiteSparseMatrixCollection krylov_problem = joinpath(dirname(pathof(Krylov)), "..", "test", "get_div_grad.jl") include(krylov_problem) -ssmc = ssmc_db() -dataset = ssmc[(ssmc.numerical_symmetry .== 1) .& (ssmc.positive_definite.== true) .& (ssmc.real .== true) .& (ssmc.nrows .≤ 100), :] +ssmc = ssmc_db(verbose=false) +dataset = ssmc[(ssmc.numerical_symmetry .== 1) .& (ssmc.positive_definite.== true) .& (ssmc.real .== true) .& (3000 .≤ ssmc.nrows .≤ 5000), :] paths = fetch_ssmc(dataset, format="MM") names = dataset[!,:name] const SUITE = BenchmarkGroup() -SUITE["CG"] = BenchmarkGroup(["CG", "SPD"]) +SUITE["Krylov"] = BenchmarkGroup() +SUITE["KrylovMethods"] = BenchmarkGroup() +SUITE["IterativeSolvers"] = BenchmarkGroup() +SUITE["KrylovKit"] = BenchmarkGroup() for N in [32, 64, 128] - SUITE["CG"]["DivGrad N=$N"] = BenchmarkGroup() A = get_div_grad(N, N, N) n, m = size(A) b = ones(n) rtol = 1.0e-6 - SUITE["CG"]["DivGrad N=$N"]["Krylov"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n) - SUITE["CG"]["DivGrad N=$N"]["KrylovMethods"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n) - SUITE["CG"]["DivGrad N=$N"]["IterativeSolvers"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n) - SUITE["CG"]["DivGrad N=$N"]["KrylovKit"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1) + SUITE["Krylov"]["DivGrad N=$N"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n) + SUITE["KrylovMethods"]["DivGrad N=$N"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n) + SUITE["IterativeSolvers"]["DivGrad N=$N"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n) + SUITE["KrylovKit"]["DivGrad N=$N"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1) end -SUITE["CG"]["UFL"] = BenchmarkGroup() nb_pbs = length(paths) for i = 1 : nb_pbs name = dataset[!,:name][i] path = paths[i] - A = Float64.(MatrixMarket.mmread(path * "/$name.mtx")) + A = MatrixMarket.mmread(path * "/$name.mtx") n, m = size(A) - b = ones(n) - rtol = 1.0e-6 - SUITE["CG"]["UFL"][name] = BenchmarkGroup() - SUITE["CG"]["UFL"][name]["Krylov"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n) - SUITE["CG"]["UFL"][name]["KrylovMethods"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n) - SUITE["CG"]["UFL"][name]["IterativeSolvers"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n) - SUITE["CG"]["UFL"][name]["KrylovKit"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1) + if eltype(A) == Float64 + b = ones(n) + rtol = 1.0e-6 + SUITE["Krylov"][name] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n) + SUITE["KrylovMethods"][name] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n) + SUITE["IterativeSolvers"][name] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n) + SUITE["KrylovKit"][name] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1) + end end diff --git a/benchmark/push_benchmarks.sh b/benchmark/push_benchmarks.sh deleted file mode 100644 index ca5fc8bfa..000000000 --- a/benchmark/push_benchmarks.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e -git clean -fd -git checkout main -git pull origin main -git fetch origin -LOCAL_BRANCH_NAME="temp_bmark" -git branch -D $LOCAL_BRANCH_NAME || true -git fetch origin pull/$pullrequest/head:$LOCAL_BRANCH_NAME -git checkout $LOCAL_BRANCH_NAME -- || true - -julia --project=benchmark -E 'using Pkg; Pkg.resolve()' -julia --project=benchmark benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c '**Starting benchmarks!**' - - -julia --project=benchmark benchmark/$1 $repo - -if [ "$?" -eq "0" ] ; then - julia --project=benchmark benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c "Benchmark results" -g "gist.json" -else - ERROR_LOGS="/home/jenkins/benchmarks/$org/$repo/${pullrequest}_${BUILD_NUMBER}_bmark_error.log" - julia --project=benchmark benchmark/send_comment_to_pr.jl -o $org -r $repo -p $pullrequest -c "**An error occured while running $1**" -g $ERROR_LOGS -fi - -git clean -fd -git reset --hard -git checkout main diff --git a/benchmark/run_benchmarks.jl b/benchmark/run_benchmarks.jl deleted file mode 100644 index 05434a28e..000000000 --- a/benchmark/run_benchmarks.jl +++ /dev/null @@ -1,85 +0,0 @@ -using Pkg -bmark_dir = @__DIR__ -repo_name = string(split(ARGS[1], ".")[1]) -bmarkname = lowercase(repo_name) - -# if we are running these benchmarks from the git repository -# we want to develop the package instead of using the release -if isdir(joinpath(bmark_dir, "..", ".git")) - Pkg.develop(PackageSpec(url = joinpath(bmark_dir, ".."))) -end - -using DataFrames -using GitHub -using JLD2 -using JSON -using PkgBenchmark -using Plots - -using SolverBenchmark - -# NB: benchmarkpkg will run benchmarks/benchmarks.jl by default -commit = benchmarkpkg(repo_name) # current state of repository -main = benchmarkpkg(repo_name, "main") -judgement = judge(commit, main) - -commit_stats = bmark_results_to_dataframes(commit) -main_stats = bmark_results_to_dataframes(main) -judgement_stats = judgement_results_to_dataframes(judgement) - -export_markdown("judgement_$(bmarkname).md", judgement) -export_markdown("main.md", main) -export_markdown("$(bmarkname).md", commit) - -function profile_solvers_from_pkgbmark(stats::Dict{Symbol,DataFrame}) - # guard against zero gctimes - costs = [ - df -> df[!, :time], - df -> df[!, :memory], - df -> df[!, :gctime] .+ 1, - df -> df[!, :allocations], - ] - profile_solvers(stats, costs, ["time", "memory", "gctime+1", "allocations"]) -end - -# extract stats for each benchmark to plot profiles -# files_dict will be part of json_dict below -files_dict = Dict{String,Any}() -file_num = 1 -for k ∈ keys(judgement_stats) - global file_num - k_stats = Dict{Symbol,DataFrame}(:commit => commit_stats[k], :main => main_stats[k]) - save_stats(k_stats, "$(bmarkname)_vs_main_$(k).jld2", force = true) - - k_profile = profile_solvers_from_pkgbmark(k_stats) - savefig("profiles_commit_vs_main_$(k).svg") - # read contents of svg file to add to gist - k_svgfile = open("profiles_commit_vs_main_$(k).svg", "r") do fd - readlines(fd) - end - # file_num makes sure svg files appear before md files (added below) - files_dict["$(file_num)_$(k).svg"] = Dict{String,Any}("content" => join(k_svgfile)) - file_num += 1 -end - -for mdfile ∈ [:judgement, :main, :commit] - global file_num - files_dict["$(file_num)_$(mdfile).md"] = - Dict{String,Any}("content" => "$(sprint(export_markdown, eval(mdfile)))") - file_num += 1 -end - -jldopen("$(bmarkname)_vs_main_judgement.jld2", "w") do file - file["jstats"] = judgement_stats -end - -# json description of gist -json_dict = Dict{String,Any}( - "description" => "$(repo_name) repository benchmark", - "public" => true, - "files" => files_dict, -) - -open("gist.json", "w") do f - JSON.print(f, json_dict) -end diff --git a/benchmark/send_comment_to_pr.jl b/benchmark/send_comment_to_pr.jl deleted file mode 100644 index f0b09fa48..000000000 --- a/benchmark/send_comment_to_pr.jl +++ /dev/null @@ -1,137 +0,0 @@ -using ArgParse -using Git -using GitHub -using JSON - -DEFAULT_GIST_FILE_PATH = "gist.json" - -function parse_commandline() - s = ArgParseSettings() - @add_arg_table! s begin - "--org", "-o" - help = "Name of GitHub Organization" - arg_type = String - default = "JuliaSmoothOptimizers" - "--repo", "-r" - help = "The name of the repository on GitHub" - arg_type = String - required = true - "--pullrequest", "-p" - help = "An integer that corresponds to the pull request" - required = true - arg_type = Int - "--gist", "-g" - help = "specify this argument if you want to send a gist to the pullrequest. If this option is not specified, you must specify a comment" - arg_type = String - required = false - "--comment", "-c" - help = "Comment to post on the pull request" - arg_type = String - required = true - end - - return parse_args(s, as_symbols = true) -end - -function create_gist_from_json_file(myauth) - gistfile = DEFAULT_GIST_FILE_PATH - gist = begin - open(gistfile, "r") do f - return JSON.parse(f) - end - end - posted_gist = create_gist(params = gist, auth = myauth) - return posted_gist -end - -function create_gist_from_log_file(gist_file, pullrequest_id, myauth) - file_content = "" - file = open(gist_file, "r") - - file_lines = readlines(file) - line_number = findfirst(x -> !isnothing(match(r"ERROR:", x)), file_lines) - lines = !isnothing(line_number) ? file_lines[line_number:end] : [""] - for line in lines - file_content *= line * '\n' - end - close(file) - - file_dict = Dict("$(pullrequest_id)_bmark_error.log" => Dict("content" => file_content)) - gist = Dict{String,Any}( - "description" => "Benchmark logs", - "public" => true, - "files" => file_dict, - ) - - posted_gist = GitHub.create_gist(params = gist, auth = myauth) - - return posted_gist -end - -function get_repo(api::GitHub.GitHubWebAPI, org::String, repo_name::String; kwargs...) - my_params = Dict(:visibility => "all") - # return GitHub.repo(api, repo; params = my_params, kwargs...) - return Repo( - GitHub.gh_get_json( - api, - "/repos/$(org)/$(repo_name)"; - params = my_params, - kwargs..., - ), - ) -end - -function get_pull_request( - api::GitHub.GitHubWebAPI, - org::String, - repo::Repo, - pullrequest_id; - kwargs..., -) - my_params = Dict(:sort => "popularity", :direction => "desc") - pull_request = PullRequest( - GitHub.gh_get_json( - api, - "/repos/$(org)/$(repo.name)/pulls/$(pullrequest_id)"; - params = my_params, - kwargs..., - ), - ) - return pull_request -end - -function post_comment_to_pr( - org::String, - repo_name::String, - pullrequest_id::Int, - comment::String; - kwargs..., -) - api = GitHub.DEFAULT_API - repo = get_repo(api, org, repo_name; kwargs...) - pull_request = get_pull_request(api, org, repo, pullrequest_id; kwargs...) - GitHub.create_comment(api, repo, pull_request, comment; kwargs...) -end - -function main() - # Need to add GITHUB_AUTH to your .bashrc - myauth = GitHub.authenticate(ENV["GITHUB_AUTH"]) - # parse the arguments: - parsed_args = parse_commandline() - org = parsed_args[:org] - repo_name = parsed_args[:repo] - pullrequest_id = parsed_args[:pullrequest] - gist_file = parsed_args[:gist] - comment = parsed_args[:comment] - - if !isnothing(gist_file) - if gist_file == DEFAULT_GIST_FILE_PATH - comment = "$(comment): $(create_gist_from_json_file(myauth).html_url)" - else - comment = "$(comment): $(create_gist_from_log_file(gist_file, pullrequest_id, myauth).html_url)" - end - end - post_comment_to_pr(org, repo_name, pullrequest_id, comment; auth = myauth) -end - -main()