From 12512710ff25bfbb3f584bb41791cbba8b40c488 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 10:53:31 +0000 Subject: [PATCH 01/19] testing Continous Benchmarking --- .github/workflows/Benchmark.yml | 54 ++++++++++++++++++++++ CIValidations/Benchmark/CMakeLists.txt | 8 ++++ CIValidations/Benchmark/MaCh3Benchmark.cpp | 38 +++++++++++++++ CIValidations/CMakeLists.txt | 4 ++ cmake/Modules/CIValidations.cmake | 5 +- 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100755 .github/workflows/Benchmark.yml create mode 100644 CIValidations/Benchmark/CMakeLists.txt create mode 100755 CIValidations/Benchmark/MaCh3Benchmark.cpp diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml new file mode 100755 index 0000000..980fd4a --- /dev/null +++ b/.github/workflows/Benchmark.yml @@ -0,0 +1,54 @@ +name: MaCh3 Benchmark + +on: + push: + branches: + - [feature_benchmark, main] + +permissions: + # Required permissions for GitHub Pages deployment and content updates + deployments: write + contents: write + +jobs: + benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - name: Get MaCh3 Validations + run: | + cd /opt/ + # Clone the MaCh3Tutorial repository with the current branch + git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + cd MaCh3Validations + mkdir build + cd build + # Configure the project with benchmark support enabled + cmake ../ -MaCh3Tutorial_Benchmark_ENABLED=TRUE + + - name: Build MaCh3 Validations + run: | + cd /opt/MaCh3Validations/build + make -j4 install # Build and install the project + + - name: Run benchmark + run: | + # Source environment setup scripts + source /opt/MaCh3Validations/build/bin/setup.MaCh3.sh + source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh + source /opt/MaCh3Validations/build/bin/setup.NuOscillator.sh + cd /opt/MaCh3Validations/build/CIValidations + # Run the benchmark and store the output in a file + ./Benchmark/MaCh3Benchmark | tee output.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'catch2' # Specify that the benchmark tool is Catch2 + github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication + output-file-path: output.txt # Path to the benchmark output file + alert-threshold: 200% # Alert threshold for performance degradation + gh-pages-branch: gh-pages # Branch to store benchmark results + gh-repository: mach3-software/MaCh3Tutorial # Repository for benchmark results + summary-always: true # Always show summary in job logs + fail-on-alert: false # Do not fail the workflow on alert diff --git a/CIValidations/Benchmark/CMakeLists.txt b/CIValidations/Benchmark/CMakeLists.txt new file mode 100644 index 0000000..0e5a87c --- /dev/null +++ b/CIValidations/Benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +foreach(TEST + MaCh3Benchmark + ) + + add_executable(${TEST} ${TEST}.cpp) + target_link_libraries(${TEST} PRIVATE Catch2::Catch2WithMain MaCh3::All MaCh3Tutorial::samplePDFTutorial) + catch_discover_tests(${TEST}) +endforeach() diff --git a/CIValidations/Benchmark/MaCh3Benchmark.cpp b/CIValidations/Benchmark/MaCh3Benchmark.cpp new file mode 100755 index 0000000..6460e30 --- /dev/null +++ b/CIValidations/Benchmark/MaCh3Benchmark.cpp @@ -0,0 +1,38 @@ +// MaCh3 spline includes +#include "mcmc/MaCh3Factory.h" +#include "samplePDF/samplePDFTutorial.h" + +#include "catch2/catch_test_macros.hpp" +#include "catch2/benchmark/catch_benchmark.hpp" + +TEST_CASE("Benchmark MaCh3") { + // Initialise manger responsible for config handling + auto FitManager = std::make_unique("Inputs/FitterConfig.yaml"); + + // Initialise covariance class reasonable for Systematics + covarianceXsec* xsec = MaCh3CovarianceFactory(FitManager.get(), "Xsec"); + covarianceOsc* osc = MaCh3CovarianceFactory(FitManager.get(), "Osc"); + + // Initialise samplePDF + auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as>(); + auto mySamples = MaCh3SamplePDFFactory(SampleConfig, xsec, osc); + + // Create MCMC Class + std::unique_ptr MaCh3Fitter = MaCh3FitterFactory(FitManager.get()); + // Add covariance to MCM + MaCh3Fitter->addSystObj(xsec); + MaCh3Fitter->addSystObj(osc); + for (size_t i = 0; i < SampleConfig.size(); ++i) { + MaCh3Fitter->addSamplePDF(mySamples[i]); + } + // Benchmark + BENCHMARK("MaCh3Fitter::DragRace") { + MaCh3Fitter->DragRace(1); + }; + + delete xsec; + delete osc; + for (size_t i = 0; i < SampleConfig.size(); ++i) { + delete mySamples[i]; + } +} diff --git a/CIValidations/CMakeLists.txt b/CIValidations/CMakeLists.txt index 9de8b3b..0e20ec0 100755 --- a/CIValidations/CMakeLists.txt +++ b/CIValidations/CMakeLists.txt @@ -39,3 +39,7 @@ install(DIRECTORY if(MaCh3Tutorial_UNITTESTS_ENABLED) add_subdirectory(UnitTests) endif() + +if(MaCh3Tutorial_Benchmark_ENABLED) + add_subdirectory(Benchmark) +endif() diff --git a/cmake/Modules/CIValidations.cmake b/cmake/Modules/CIValidations.cmake index ce7f232..5af2b61 100755 --- a/cmake/Modules/CIValidations.cmake +++ b/cmake/Modules/CIValidations.cmake @@ -1,5 +1,6 @@ option(MaCh3Tutorial_UNITTESTS_ENABLED "Whether to build MaCh3 Unit Tests" OFF) -option(MaCh3Tutorial_Coverage_ENABLED "Whether to build MaCh3 Coverage " OFF) +option(MaCh3Tutorial_Coverage_ENABLED "Whether to build MaCh3 Coverage" OFF) +option(MaCh3Tutorial_Benchmark_ENABLED "Enable benchmarking" ON) # WARNING FIXME CPMAddPackage( NAME NuMCMCTools @@ -14,7 +15,7 @@ install(DIRECTORY DESTINATION ${CMAKE_BINARY_DIR}/NuMCMCTools) ############################ Catch2/CTest #################################### -if(MaCh3Tutorial_UNITTESTS_ENABLED) +if(MaCh3Tutorial_UNITTESTS_ENABLED OR MaCh3Tutorial_Benchmark_ENABLED) find_package(Catch2 QUIET) if(NOT Catch2_FOUND) CPMAddPackage("gh:catchorg/Catch2@3.5.2") From be7448b082ee79aab9ca5efbd5103de4bb45606b Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 10:56:26 +0000 Subject: [PATCH 02/19] now? --- .github/workflows/Benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 980fd4a..11bbfc3 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -3,7 +3,8 @@ name: MaCh3 Benchmark on: push: branches: - - [feature_benchmark, main] + - feature_benchmark + - main permissions: # Required permissions for GitHub Pages deployment and content updates From 5741b7045b5cdd983b91dd10bdd5953b912c7fdd Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 10:59:59 +0000 Subject: [PATCH 03/19] now? --- .github/workflows/Benchmark.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 11bbfc3..5ec7181 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -3,8 +3,7 @@ name: MaCh3 Benchmark on: push: branches: - - feature_benchmark - - main + - [feature_benchmark, main] permissions: # Required permissions for GitHub Pages deployment and content updates @@ -20,7 +19,9 @@ jobs: run: | cd /opt/ # Clone the MaCh3Tutorial repository with the current branch - git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + # FIXME BLLLLLLARB@@@@@@ + git clone --branch feature_benchmark https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + #git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations cd MaCh3Validations mkdir build cd build @@ -47,7 +48,7 @@ jobs: with: tool: 'catch2' # Specify that the benchmark tool is Catch2 github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication - output-file-path: output.txt # Path to the benchmark output file + output-file-path: /opt/MaCh3Validations/build/CIValidations/output.txt # Path to the benchmark output file alert-threshold: 200% # Alert threshold for performance degradation gh-pages-branch: gh-pages # Branch to store benchmark results gh-repository: mach3-software/MaCh3Tutorial # Repository for benchmark results From 937614b4079be0ab3bb360b26a434ccae96af353 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:02:37 +0000 Subject: [PATCH 04/19] :( --- .github/workflows/Benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 5ec7181..001c822 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -3,7 +3,8 @@ name: MaCh3 Benchmark on: push: branches: - - [feature_benchmark, main] + - feature_benchmark + - main permissions: # Required permissions for GitHub Pages deployment and content updates From 4d626662ce989dec98943ab66687c41bb05d46db Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:03:59 +0000 Subject: [PATCH 05/19] :( --- .github/workflows/Benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 001c822..cd4fc9a 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -1,3 +1,4 @@ +--- name: MaCh3 Benchmark on: From c47b416d5cfbf2d0032634e7327d613cfff573d6 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:05:39 +0000 Subject: [PATCH 06/19] closer --- .github/workflows/Benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index cd4fc9a..68adc76 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -15,7 +15,8 @@ permissions: jobs: benchmark: name: Performance regression check - runs-on: ubuntu-latest + container: + image: ghcr.io/mach3-software/mach3:alma9v1.3.0 steps: - name: Get MaCh3 Validations run: | From 2ba500318919263dab9a4426f224c7b900e45d69 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:08:03 +0000 Subject: [PATCH 07/19] closer --- .github/workflows/Benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 68adc76..7f9f411 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -15,6 +15,7 @@ permissions: jobs: benchmark: name: Performance regression check + runs-on: ubuntu-latest container: image: ghcr.io/mach3-software/mach3:alma9v1.3.0 steps: From 0c8c74b16629d92ae192940a913ea98d873841a7 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:10:09 +0000 Subject: [PATCH 08/19] closer --- .github/workflows/Benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 7f9f411..d3ff079 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -30,7 +30,7 @@ jobs: mkdir build cd build # Configure the project with benchmark support enabled - cmake ../ -MaCh3Tutorial_Benchmark_ENABLED=TRUE + cmake ../ -DMaCh3Tutorial_Benchmark_ENABLED=TRUE - name: Build MaCh3 Validations run: | From e8fab8834f1b71373cbd23442121d12897947ddd Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 11:23:55 +0000 Subject: [PATCH 09/19] closer --- .github/workflows/Benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index d3ff079..abf2bbe 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -43,16 +43,16 @@ jobs: source /opt/MaCh3Validations/build/bin/setup.MaCh3.sh source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh source /opt/MaCh3Validations/build/bin/setup.NuOscillator.sh - cd /opt/MaCh3Validations/build/CIValidations + cd /opt/MaCh3Validations/build/ # Run the benchmark and store the output in a file - ./Benchmark/MaCh3Benchmark | tee output.txt + ./CIValidations/Benchmark/MaCh3Benchmark | tee output.txt - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: tool: 'catch2' # Specify that the benchmark tool is Catch2 github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication - output-file-path: /opt/MaCh3Validations/build/CIValidations/output.txt # Path to the benchmark output file + output-file-path: /opt/MaCh3Validations/build/output.txt # Path to the benchmark output file alert-threshold: 200% # Alert threshold for performance degradation gh-pages-branch: gh-pages # Branch to store benchmark results gh-repository: mach3-software/MaCh3Tutorial # Repository for benchmark results From b2e8cef3087fcb91c50992b09ce819661919d1b7 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 12:48:49 +0000 Subject: [PATCH 10/19] let's try another attempt --- CMakeLists.txt | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7972b3..64d819b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,8 @@ file( ) include(${CMAKE_CURRENT_BINARY_DIR}/cmake/CPM.cmake) +include(CIValidations) + ################################## MaCh3 ###################################### # KS: Here we try to find tag matching tutorial version. If we can't find one then use develop # This will allow to grab tutorial for tagged MaCh3 version without a need of manually changing version @@ -67,13 +69,27 @@ if(NOT DEFINED PYTHON_ENABLED) SET(PYTHON_ENABLED false) endif() if(NOT MaCh3_FOUND) + + set(MaCh3_OPTIONS + "MaCh3_PYTHON_ENABLED ${PYTHON_ENABLED}" + ) + + # Add LOG_LEVEL if defined + if (DEFINED LOG_LEVEL) + list(APPEND MaCh3_OPTIONS "LOG_LEVEL ${LOG_LEVEL}") + endif() + + if (MaCh3Tutorial_Benchmark_ENABLED) + list(APPEND MaCh3_OPTIONS "MaCh3_WERROR_ENABLED OFF") + endif() + CPMAddPackage( NAME MaCh3 GIT_TAG ${MaCh3_Branch} GIT_SHALLOW YES GITHUB_REPOSITORY mach3-software/MaCh3 OPTIONS - "MaCh3_PYTHON_ENABLED ${PYTHON_ENABLED}" + ${MaCh3_OPTIONS} "Prob3ppLinear_ENABLED ON" "NuFastLinear_ENABLED ON" ) @@ -92,8 +108,6 @@ if(MaCh3_VERSION GREATER MaCh3Tutorial_VERSION) cmessage(WARNING "MaCh3Tutorial_VERSION (${MaCh3Tutorial_VERSION}) is smaller than MaCh3_VERSION (${MaCh3_VERSION}), this most likely will not work") endif() -include(CIValidations) - ############################ C++ Compiler #################################### get_property(MACH3_CXX_STANDARD GLOBAL PROPERTY MACH3_CXX_STANDARD) if(NOT DEFINED MACH3_CXX_STANDARD) From ccb6b1e2d96f1e8db0fa58921b4929df4684d65d Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 12:57:24 +0000 Subject: [PATCH 11/19] forgot to turn off logger... --- CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64d819b..5862819 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,15 +74,18 @@ if(NOT MaCh3_FOUND) "MaCh3_PYTHON_ENABLED ${PYTHON_ENABLED}" ) + # KS: For benchmark we turn off logger to not interfere with output messages. + # Also to have code compile let's turn off Werror + if (MaCh3Tutorial_Benchmark_ENABLED) + list(APPEND MaCh3_OPTIONS "MaCh3_WERROR_ENABLED OFF") + set(LOG_LEVEL "OFF") + endif() + # Add LOG_LEVEL if defined if (DEFINED LOG_LEVEL) list(APPEND MaCh3_OPTIONS "LOG_LEVEL ${LOG_LEVEL}") endif() - if (MaCh3Tutorial_Benchmark_ENABLED) - list(APPEND MaCh3_OPTIONS "MaCh3_WERROR_ENABLED OFF") - endif() - CPMAddPackage( NAME MaCh3 GIT_TAG ${MaCh3_Branch} From d99f8811ec91067d5f1a14641bced3e01eee9d6e Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 13:05:20 +0000 Subject: [PATCH 12/19] fix repository name --- .github/workflows/Benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index abf2bbe..58be45c 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -55,6 +55,6 @@ jobs: output-file-path: /opt/MaCh3Validations/build/output.txt # Path to the benchmark output file alert-threshold: 200% # Alert threshold for performance degradation gh-pages-branch: gh-pages # Branch to store benchmark results - gh-repository: mach3-software/MaCh3Tutorial # Repository for benchmark results + gh-repository: https://github.com/mach3-software/MaCh3Tutorial.git # Repository for benchmark results summary-always: true # Always show summary in job logs fail-on-alert: false # Do not fail the workflow on alert From 6e5ee49d43ad9242f7f70b07ecef91e27eecf80a Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 13:13:53 +0000 Subject: [PATCH 13/19] fix repository name --- .github/workflows/Benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 58be45c..254a0b2 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -55,6 +55,6 @@ jobs: output-file-path: /opt/MaCh3Validations/build/output.txt # Path to the benchmark output file alert-threshold: 200% # Alert threshold for performance degradation gh-pages-branch: gh-pages # Branch to store benchmark results - gh-repository: https://github.com/mach3-software/MaCh3Tutorial.git # Repository for benchmark results + gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs fail-on-alert: false # Do not fail the workflow on alert From f2491b0ad8d74d8878bb6e15c1e663c16fa9d7f0 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 13:43:54 +0000 Subject: [PATCH 14/19] forgot autopush... --- .github/workflows/Benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 254a0b2..7935a5e 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -58,3 +58,5 @@ jobs: gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs fail-on-alert: false # Do not fail the workflow on alert + # Push and deploy GitHub pages branch automatically + auto-push: true From 0462ee8fa0d5484d591ff2bb6f5e9d87e593f41b Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 13:52:30 +0000 Subject: [PATCH 15/19] use ROOT directory --- .github/workflows/Benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 7935a5e..658705c 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -58,5 +58,5 @@ jobs: gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs fail-on-alert: false # Do not fail the workflow on alert - # Push and deploy GitHub pages branch automatically - auto-push: true + auto-push: true # Push and deploy GitHub pages branch automatically + benchmark-data-dir-path: "" # Use ROOT directory From 1839272e3b95e340a7b565ae59eef51f7404a39e Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 14:36:17 +0000 Subject: [PATCH 16/19] tidy before PR --- .github/workflows/Benchmark.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 658705c..3b1d7c9 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -2,10 +2,10 @@ name: MaCh3 Benchmark on: + pull_request: + branches: [ main ] push: - branches: - - feature_benchmark - - main + branches: [ main ] permissions: # Required permissions for GitHub Pages deployment and content updates @@ -23,9 +23,12 @@ jobs: run: | cd /opt/ # Clone the MaCh3Tutorial repository with the current branch - # FIXME BLLLLLLARB@@@@@@ - git clone --branch feature_benchmark https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations - #git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + # Fix Me + if [ "${{ github.event_name }}" == 'pull_request' ]; then + git clone --branch feature_benchmark https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + else + git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations + fi cd MaCh3Validations mkdir build cd build @@ -58,5 +61,6 @@ jobs: gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs fail-on-alert: false # Do not fail the workflow on alert - auto-push: true # Push and deploy GitHub pages branch automatically benchmark-data-dir-path: "" # Use ROOT directory + # Push and deploy GitHub pages branch automatically, only if type isn't PR + auto-push: ${{ github.event_name != 'pull_request' }} From fbab6cdeea76e4599bb599f4e4f6c7296835c581 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 14:45:00 +0000 Subject: [PATCH 17/19] test if fails --- .github/workflows/Benchmark.yml | 3 ++- CIValidations/Benchmark/MaCh3Benchmark.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 3b1d7c9..7b40bdc 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -57,10 +57,11 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication output-file-path: /opt/MaCh3Validations/build/output.txt # Path to the benchmark output file alert-threshold: 200% # Alert threshold for performance degradation + comment-on-alert: true gh-pages-branch: gh-pages # Branch to store benchmark results gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs - fail-on-alert: false # Do not fail the workflow on alert + fail-on-alert: true # Do not fail the workflow on alert benchmark-data-dir-path: "" # Use ROOT directory # Push and deploy GitHub pages branch automatically, only if type isn't PR auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/CIValidations/Benchmark/MaCh3Benchmark.cpp b/CIValidations/Benchmark/MaCh3Benchmark.cpp index 6460e30..694d9de 100755 --- a/CIValidations/Benchmark/MaCh3Benchmark.cpp +++ b/CIValidations/Benchmark/MaCh3Benchmark.cpp @@ -27,7 +27,7 @@ TEST_CASE("Benchmark MaCh3") { } // Benchmark BENCHMARK("MaCh3Fitter::DragRace") { - MaCh3Fitter->DragRace(1); + MaCh3Fitter->DragRace(1000); }; delete xsec; From e9896ceb92ef564eccfe8981db101cf383a93321 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 14:56:09 +0000 Subject: [PATCH 18/19] final push --- .github/workflows/Benchmark.yml | 2 +- CIValidations/Benchmark/MaCh3Benchmark.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml index 7b40bdc..69673fe 100755 --- a/.github/workflows/Benchmark.yml +++ b/.github/workflows/Benchmark.yml @@ -61,7 +61,7 @@ jobs: gh-pages-branch: gh-pages # Branch to store benchmark results gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results summary-always: true # Always show summary in job logs - fail-on-alert: true # Do not fail the workflow on alert + fail-on-alert: false # Do not fail the workflow on alert benchmark-data-dir-path: "" # Use ROOT directory # Push and deploy GitHub pages branch automatically, only if type isn't PR auto-push: ${{ github.event_name != 'pull_request' }} diff --git a/CIValidations/Benchmark/MaCh3Benchmark.cpp b/CIValidations/Benchmark/MaCh3Benchmark.cpp index 694d9de..6460e30 100755 --- a/CIValidations/Benchmark/MaCh3Benchmark.cpp +++ b/CIValidations/Benchmark/MaCh3Benchmark.cpp @@ -27,7 +27,7 @@ TEST_CASE("Benchmark MaCh3") { } // Benchmark BENCHMARK("MaCh3Fitter::DragRace") { - MaCh3Fitter->DragRace(1000); + MaCh3Fitter->DragRace(1); }; delete xsec; From 8fe1b8edba3c1873e850783944ea2333ef0a5c15 Mon Sep 17 00:00:00 2001 From: Kamil Skwarczynski Date: Mon, 6 Jan 2025 14:56:53 +0000 Subject: [PATCH 19/19] forogor to dleeete --- .github/workflows/Benchmark.yml | 67 --------------------------------- 1 file changed, 67 deletions(-) delete mode 100755 .github/workflows/Benchmark.yml diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml deleted file mode 100755 index 69673fe..0000000 --- a/.github/workflows/Benchmark.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -name: MaCh3 Benchmark - -on: - pull_request: - branches: [ main ] - push: - branches: [ main ] - -permissions: - # Required permissions for GitHub Pages deployment and content updates - deployments: write - contents: write - -jobs: - benchmark: - name: Performance regression check - runs-on: ubuntu-latest - container: - image: ghcr.io/mach3-software/mach3:alma9v1.3.0 - steps: - - name: Get MaCh3 Validations - run: | - cd /opt/ - # Clone the MaCh3Tutorial repository with the current branch - # Fix Me - if [ "${{ github.event_name }}" == 'pull_request' ]; then - git clone --branch feature_benchmark https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations - else - git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations - fi - cd MaCh3Validations - mkdir build - cd build - # Configure the project with benchmark support enabled - cmake ../ -DMaCh3Tutorial_Benchmark_ENABLED=TRUE - - - name: Build MaCh3 Validations - run: | - cd /opt/MaCh3Validations/build - make -j4 install # Build and install the project - - - name: Run benchmark - run: | - # Source environment setup scripts - source /opt/MaCh3Validations/build/bin/setup.MaCh3.sh - source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh - source /opt/MaCh3Validations/build/bin/setup.NuOscillator.sh - cd /opt/MaCh3Validations/build/ - # Run the benchmark and store the output in a file - ./CIValidations/Benchmark/MaCh3Benchmark | tee output.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'catch2' # Specify that the benchmark tool is Catch2 - github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication - output-file-path: /opt/MaCh3Validations/build/output.txt # Path to the benchmark output file - alert-threshold: 200% # Alert threshold for performance degradation - comment-on-alert: true - gh-pages-branch: gh-pages # Branch to store benchmark results - gh-repository: github.com/mach3-software/MaCh3Tutorial # Repository for benchmark results - summary-always: true # Always show summary in job logs - fail-on-alert: false # Do not fail the workflow on alert - benchmark-data-dir-path: "" # Use ROOT directory - # Push and deploy GitHub pages branch automatically, only if type isn't PR - auto-push: ${{ github.event_name != 'pull_request' }}