From a52ad1595a1530d896317185eb2c077a66d0b7e2 Mon Sep 17 00:00:00 2001 From: Tangi Migot Date: Wed, 3 Jul 2024 00:14:52 +0200 Subject: [PATCH] Add gradient benchmark CI (#261) --- .github/workflows/BenchmarkGradient.yml | 30 +++++++++++++++ .gitignore | 4 +- benchmark/Manifest.toml | 50 +++++++++++++------------ benchmark/benchmarks_grad.jl | 16 ++++++++ 4 files changed, 76 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/BenchmarkGradient.yml create mode 100644 benchmark/benchmarks_grad.jl diff --git a/.github/workflows/BenchmarkGradient.yml b/.github/workflows/BenchmarkGradient.yml new file mode 100644 index 00000000..b135be2a --- /dev/null +++ b/.github/workflows/BenchmarkGradient.yml @@ -0,0 +1,30 @@ +name: Run gradient benchmarks + +on: + pull_request: + types: [labeled, opened, synchronize, reopened] + +# Only trigger the benchmark job when you add `run benchmark` label to the PR +jobs: + Benchmark: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'run gradient benchmark') + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@latest + with: + version: 1 + - uses: julia-actions/julia-buildpkg@latest + - name: Install dependencies + run: julia -e 'using Pkg; pkg"add PkgBenchmark BenchmarkCI@0.1"' + - name: Run benchmarks + run: julia -e 'using BenchmarkCI; BenchmarkCI.judge(;baseline = "origin/main", script = joinpath(pwd(), "benchmark", "benchmarks_grad.jl"))' + - name: Post results + run: julia -e 'using BenchmarkCI; BenchmarkCI.postjudge()' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Push results + run: julia -e "using BenchmarkCI; BenchmarkCI.pushresult(;branch = "benchmark-gradients-results")" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SSH_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.gitignore b/.gitignore index e0791f30..33dcb6f9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ *.jl.mem docs/build docs/site -Manifest.toml \ No newline at end of file +Manifest.toml +/.benchmarkci +/benchmark/*.json diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml index ad9a0b7b..486f0e57 100644 --- a/benchmark/Manifest.toml +++ b/benchmark/Manifest.toml @@ -1,14 +1,12 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.1" +julia_version = "1.10.2" manifest_format = "2.0" project_hash = "0cb0756144aac73ae8e2d06d9a0f6567a7a2f964" [[deps.ADNLPModels]] deps = ["ADTypes", "ForwardDiff", "LinearAlgebra", "NLPModels", "Requires", "ReverseDiff", "SparseArrays", "SparseConnectivityTracer", "SparseMatrixColorings"] -git-tree-sha1 = "2b582670fb51216d8d000c6de72934d1f68c4e7c" -repo-rev = "main" -repo-url = "https://github.com/JuliaSmoothOptimizers/ADNLPModels.jl" +path = ".." uuid = "54578032-b7ea-4c30-94aa-7cbd1cce6c9a" version = "0.8.2" @@ -251,7 +249,7 @@ weakdeps = ["Dates", "LinearAlgebra"] [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.2+0" +version = "1.1.0+0" [[deps.CompositeTypes]] git-tree-sha1 = "bce26c3dab336582805503bed209faab1c279768" @@ -844,21 +842,26 @@ version = "0.2.0" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" +version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" +version = "8.4.0+0" [[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" +version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -987,7 +990,7 @@ version = "1.1.9" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+0" +version = "2.28.2+1" [[deps.Measures]] git-tree-sha1 = "c13304c81eec1ed3af7fc20e75fb6b26092a1102" @@ -1005,7 +1008,7 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.10.11" +version = "2023.1.10" [[deps.MultivariatePolynomials]] deps = ["ChainRulesCore", "DataStructures", "LinearAlgebra", "MutableArithmetics"] @@ -1056,12 +1059,12 @@ version = "1.3.5+1" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.21+4" +version = "0.3.23+4" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" -version = "0.8.1+0" +version = "0.8.1+2" [[deps.OpenSSL]] deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] @@ -1101,7 +1104,7 @@ version = "1.6.3" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.42.0+0" +version = "10.42.0+1" [[deps.PDMats]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] @@ -1153,7 +1156,7 @@ version = "0.43.4+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.0" +version = "1.10.0" [[deps.PkgBenchmark]] deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Pkg", "Printf", "TerminalLoggers", "UUIDs"] @@ -1269,7 +1272,7 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] -deps = ["SHA", "Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.RandomExtensions]] @@ -1437,6 +1440,7 @@ version = "1.2.1" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" [[deps.SparseConnectivityTracer]] deps = ["ADTypes", "Compat", "DocStringExtensions", "Random", "Requires", "SparseArrays"] @@ -1499,7 +1503,7 @@ version = "1.4.3" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.9.0" +version = "1.10.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -1557,9 +1561,9 @@ deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "5.10.1+6" +version = "7.2.1+1" [[deps.SymbolicIndexingInterface]] git-tree-sha1 = "be414bfd80c2c91197823890c66ef4b74f5bf5fe" @@ -1917,7 +1921,7 @@ version = "1.5.0+0" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+0" +version = "1.2.13+1" [[deps.Zstd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1980,7 +1984,7 @@ version = "0.15.1+0" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.8.0+0" +version = "5.8.0+1" [[deps.libevdev_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -2027,12 +2031,12 @@ version = "1.1.6+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" +version = "1.52.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+0" +version = "17.4.0+2" [[deps.x264_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] diff --git a/benchmark/benchmarks_grad.jl b/benchmark/benchmarks_grad.jl new file mode 100644 index 00000000..5e206ede --- /dev/null +++ b/benchmark/benchmarks_grad.jl @@ -0,0 +1,16 @@ +# Include useful packages +using ADNLPModels +using Dates, DelimitedFiles, JLD2, LinearAlgebra, Printf, SparseArrays +using BenchmarkTools, DataFrames +#JSO packages +using NLPModels, OptimizationProblems, SolverBenchmark +# Most likely benchmark with JuMP as well +using JuMP, NLPModelsJuMP + +include("problems_sets.jl") +verbose_subbenchmark = false + +# Run locally with `tune!(SUITE)` and then `run(SUITE)` +const SUITE = BenchmarkGroup() + +include("gradient/benchmarks_gradient.jl")