testing Continous Benchmarking

Author: KSkwarczynski

.github/workflows/Benchmark.yml

@@ -0,0 +1,54 @@
+name: MaCh3 Benchmark
+
+on:
+  push:
+    branches:
+      - [feature_benchmark, main]
+
+permissions:
+  # Required permissions for GitHub Pages deployment and content updates
+  deployments: write
+  contents: write
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get MaCh3 Validations
+        run: |
+          cd /opt/
+          # Clone the MaCh3Tutorial repository with the current branch
+          git clone --branch "${{ github.head_ref }}" https://github.com/mach3-software/MaCh3Tutorial.git MaCh3Validations
+          cd MaCh3Validations
+          mkdir build
+          cd build
+          # Configure the project with benchmark support enabled
+          cmake ../ -MaCh3Tutorial_Benchmark_ENABLED=TRUE
+
+      - name: Build MaCh3 Validations
+        run: |
+          cd /opt/MaCh3Validations/build
+          make -j4 install # Build and install the project
+
+      - name: Run benchmark
+        run: |
+          # Source environment setup scripts
+          source /opt/MaCh3Validations/build/bin/setup.MaCh3.sh
+          source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh
+          source /opt/MaCh3Validations/build/bin/setup.NuOscillator.sh
+          cd /opt/MaCh3Validations/build/CIValidations
+          # Run the benchmark and store the output in a file
+          ./Benchmark/MaCh3Benchmark | tee output.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'catch2' # Specify that the benchmark tool is Catch2
+          github-token: ${{ secrets.GITHUB_TOKEN }} # Use GitHub token for authentication
+          output-file-path: output.txt # Path to the benchmark output file
+          alert-threshold: 200% # Alert threshold for performance degradation
+          gh-pages-branch: gh-pages # Branch to store benchmark results
+          gh-repository: mach3-software/MaCh3Tutorial # Repository for benchmark results
+          summary-always: true # Always show summary in job logs
+          fail-on-alert: false # Do not fail the workflow on alert

CIValidations/Benchmark/CMakeLists.txt

@@ -0,0 +1,8 @@
+foreach(TEST
+    MaCh3Benchmark
+    )
+
+    add_executable(${TEST} ${TEST}.cpp)
+    target_link_libraries(${TEST} PRIVATE Catch2::Catch2WithMain MaCh3::All MaCh3Tutorial::samplePDFTutorial)
+    catch_discover_tests(${TEST})
+endforeach()

CIValidations/Benchmark/MaCh3Benchmark.cpp

@@ -0,0 +1,38 @@
+// MaCh3 spline includes
+#include "mcmc/MaCh3Factory.h"
+#include "samplePDF/samplePDFTutorial.h"
+
+#include "catch2/catch_test_macros.hpp"
+#include "catch2/benchmark/catch_benchmark.hpp"
+
+TEST_CASE("Benchmark MaCh3") {
+  // Initialise manger responsible for config handling
+  auto FitManager = std::make_unique<manager>("Inputs/FitterConfig.yaml");
+
+  // Initialise covariance class reasonable for Systematics
+  covarianceXsec* xsec = MaCh3CovarianceFactory(FitManager.get(), "Xsec");
+  covarianceOsc*  osc  = MaCh3CovarianceFactory<covarianceOsc>(FitManager.get(), "Osc");
+
+  // Initialise samplePDF
+  auto SampleConfig = FitManager->raw()["General"]["TutorialSamples"].as<std::vector<std::string>>();
+  auto mySamples = MaCh3SamplePDFFactory<samplePDFTutorial>(SampleConfig, xsec, osc);
+
+  // Create MCMC Class
+  std::unique_ptr<FitterBase> MaCh3Fitter = MaCh3FitterFactory(FitManager.get());
+  // Add covariance to MCM
+  MaCh3Fitter->addSystObj(xsec);
+  MaCh3Fitter->addSystObj(osc);
+  for (size_t i = 0; i < SampleConfig.size(); ++i) {
+    MaCh3Fitter->addSamplePDF(mySamples[i]);
+  }
+  // Benchmark
+  BENCHMARK("MaCh3Fitter::DragRace") {
+    MaCh3Fitter->DragRace(1);
+  };
+
+  delete xsec;
+  delete osc;
+  for (size_t i = 0; i < SampleConfig.size(); ++i) {
+    delete mySamples[i];
+  }
+}

CIValidations/CMakeLists.txt

@@ -39,3 +39,7 @@ install(DIRECTORY
 if(MaCh3Tutorial_UNITTESTS_ENABLED)
   add_subdirectory(UnitTests)
 endif()
+
+if(MaCh3Tutorial_Benchmark_ENABLED)
+  add_subdirectory(Benchmark)
+endif()

cmake/Modules/CIValidations.cmake

@@ -1,5 +1,6 @@
 option(MaCh3Tutorial_UNITTESTS_ENABLED "Whether to build MaCh3 Unit Tests" OFF)
-option(MaCh3Tutorial_Coverage_ENABLED "Whether to build MaCh3 Coverage " OFF)
+option(MaCh3Tutorial_Coverage_ENABLED "Whether to build MaCh3 Coverage" OFF)
+option(MaCh3Tutorial_Benchmark_ENABLED "Enable benchmarking" ON) # WARNING FIXME
 
 CPMAddPackage(
   NAME NuMCMCTools
@@ -14,7 +15,7 @@ install(DIRECTORY
     DESTINATION ${CMAKE_BINARY_DIR}/NuMCMCTools)
 
 ############################  Catch2/CTest  ####################################
-if(MaCh3Tutorial_UNITTESTS_ENABLED)
+if(MaCh3Tutorial_UNITTESTS_ENABLED OR MaCh3Tutorial_Benchmark_ENABLED)
   find_package(Catch2 QUIET)
   if(NOT Catch2_FOUND)
     CPMAddPackage("gh:catchorg/Catch2@3.5.2")