Update benchmarks

Jenkinsfile → benchmark/Jenkinsfile

@@ -1,4 +1,4 @@
-def bmarkFile = 'run_benchmarks.jl'
+def bmarkFile = 'benchmarks.jl'
 def prNumber = BRANCH_NAME.tokenize("PR-")[0]
 pipeline {
   agent any
@@ -87,11 +87,25 @@ pipeline {
             git checkout $LOCAL_BRANCH_NAME --
           '''
         }
+        sh "git clone https://github.com/ProofOfConceptForJuliSmoothOptimizers/BenchmarkSetup.git || true"
+      }
+    }
+    stage('setup benchmark repository') {
+      when {
+        expression { env.comment }
+      }
+      steps {
+        dir(WORKSPACE + "/BenchmarkSetup") {
+          sh '''
+            git fetch origin
+            git pull --ff-only origin master
+          '''
+        }   
       }
     }
     stage('run benchmarks') {
       when {
-        expression {env.comment}
+        expression { env.comment }
       }
       steps {
         script {
@@ -101,9 +115,9 @@ pipeline {
           }
         }
         dir(WORKSPACE + "/$repo") {
-          sh "mkdir -p $HOME/benchmarks/${org}/${repo}"
-          sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log benchmark/push_benchmarks.sh $bmarkFile"  
-        }
+        sh "mkdir -p $HOME/benchmarks/${org}/${repo}"
+        sh "qsub -N ${repo}_${pullrequest} -V -cwd -o $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_output.log -e $HOME/benchmarks/${org}/${repo}/${pullrequest}_${BUILD_NUMBER}_bmark_error.log ../BenchmarkSetup/benchmark/push_benchmarks.sh ${bmarkFile}"
+        }   
       }
     }
   }
@@ -116,4 +130,3 @@ pipeline {
     }
   }
 }
-

benchmark/Project.toml

@@ -13,8 +13,10 @@ KrylovMethods = "9a2cd570-f05c-5dc1-9209-93ad6f5727f7"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MatrixMarket = "4d4711f2-db25-561a-b6b3-d35e7d4047d3"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+PlotThemes = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee"
 SolverBenchmark = "581a75fa-a23a-52d0-a590-d6201de2218a"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 SuiteSparseMatrixCollection = "ac199af8-68bc-55b8-82c4-7abd6f96ed98"

benchmark/cg_bmark.jl

@@ -14,39 +14,41 @@ using SuiteSparseMatrixCollection
 krylov_problem = joinpath(dirname(pathof(Krylov)), "..", "test", "get_div_grad.jl")
 include(krylov_problem)
 
-ssmc = ssmc_db()
-dataset = ssmc[(ssmc.numerical_symmetry .== 1) .& (ssmc.positive_definite.== true) .& (ssmc.real .== true) .& (ssmc.nrows .≤ 100), :]
+ssmc = ssmc_db(verbose=false)
+dataset = ssmc[(ssmc.numerical_symmetry .== 1) .& (ssmc.positive_definite.== true) .& (ssmc.real .== true) .& (3000 .≤ ssmc.nrows .≤ 5000), :]
 paths = fetch_ssmc(dataset, format="MM")
 names = dataset[!,:name]
 
 const SUITE = BenchmarkGroup()
 
-SUITE["CG"] = BenchmarkGroup(["CG", "SPD"])
+SUITE["Krylov"] = BenchmarkGroup()
+SUITE["KrylovMethods"] = BenchmarkGroup()
+SUITE["IterativeSolvers"] = BenchmarkGroup()
+SUITE["KrylovKit"] = BenchmarkGroup()
 
 for N in [32, 64, 128]
-  SUITE["CG"]["DivGrad N=$N"] = BenchmarkGroup()
   A = get_div_grad(N, N, N)
   n, m = size(A)
   b = ones(n)
   rtol = 1.0e-6
-  SUITE["CG"]["DivGrad N=$N"]["Krylov"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n)
-  SUITE["CG"]["DivGrad N=$N"]["KrylovMethods"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n)
-  SUITE["CG"]["DivGrad N=$N"]["IterativeSolvers"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n)
-  SUITE["CG"]["DivGrad N=$N"]["KrylovKit"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1)
+  SUITE["Krylov"]["DivGrad N=$N"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n)
+  SUITE["KrylovMethods"]["DivGrad N=$N"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n)
+  SUITE["IterativeSolvers"]["DivGrad N=$N"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n)
+  SUITE["KrylovKit"]["DivGrad N=$N"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1)
 end
 
-SUITE["CG"]["UFL"] = BenchmarkGroup()
 nb_pbs = length(paths)
 for i = 1 : nb_pbs
   name = dataset[!,:name][i]
   path = paths[i]
-  A = Float64.(MatrixMarket.mmread(path * "/$name.mtx"))
+  A = MatrixMarket.mmread(path * "/$name.mtx")
   n, m = size(A)
-  b = ones(n)
-  rtol = 1.0e-6
-  SUITE["CG"]["UFL"][name] = BenchmarkGroup()
-  SUITE["CG"]["UFL"][name]["Krylov"] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n)
-  SUITE["CG"]["UFL"][name]["KrylovMethods"] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n)
-  SUITE["CG"]["UFL"][name]["IterativeSolvers"] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n)
-  SUITE["CG"]["UFL"][name]["KrylovKit"] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1)
+  if eltype(A) == Float64
+    b = ones(n)
+    rtol = 1.0e-6
+    SUITE["Krylov"][name] = @benchmarkable cg($A, $b, atol=0.0, rtol=$rtol, itmax=$n)
+    SUITE["KrylovMethods"][name] = @benchmarkable ($KrylovMethods.cg)($A, $b, tol=$rtol, maxIter=$n)
+    SUITE["IterativeSolvers"][name] = @benchmarkable ($IterativeSolvers.cg)($A, $b, abstol=0.0, reltol=$rtol, maxiter=$n)
+    SUITE["KrylovKit"][name] = @benchmarkable ($KrylovKit.linsolve)($A, $b, atol=0.0, rtol=$rtol, isposdef=true, krylovdim=$n, maxiter=1)
+  end
 end