From 02c3692659398c694c1f57b20c0a2922b94f7179 Mon Sep 17 00:00:00 2001 From: BrandonPacewic <92102436+BrandonPacewic@users.noreply.github.com> Date: Fri, 18 Apr 2025 11:30:06 -0700 Subject: [PATCH 1/3] feat(bench): add bench output scripts --- Makefile | 8 ++++ tools/scripts/compare_benchmarks.py | 59 ++++++++++++++++++++++++++++ tools/scripts/output_benchmarks.py | 61 +++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 tools/scripts/compare_benchmarks.py create mode 100644 tools/scripts/output_benchmarks.py diff --git a/Makefile b/Makefile index ca4cbc7..8164cc4 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,12 @@ bench: build run-benchmarks: bench @cmake --build "$(BUILD_DIR)" --target run_all_benchmarks -- -j$(JOBS) +bench-diff: + @python tools/scripts/compare_benchmarks.py + +bench-out: + @python tools/scripts/output_benchmarks.py + tools: BUILD_TOOLS := ON tools: build @@ -54,6 +60,8 @@ help: @echo " test - run ctest (RUN_TESTS=ON)" @echo " bench - build & run benchmarks (BUILD_BENCH=ON)" @echo " run-benchmarks - run all benchmarks" + @echo " bench-diff - compare the two most recent benchmark runs" + @echo " bench-out - output the most recent benchmark run" @echo " tools - build project-defined tools (BUILD_TOOLS=ON)" @echo " format - run clang-format on all source files" @echo " clean - clean build artefacts" diff --git a/tools/scripts/compare_benchmarks.py b/tools/scripts/compare_benchmarks.py new file mode 100644 index 0000000..31e611c --- /dev/null +++ b/tools/scripts/compare_benchmarks.py @@ -0,0 +1,59 @@ +# Copyright (c) Brandon Pacewic +# SPDX-License-Identifier: MIT + +import os +import re +import subprocess +from pathlib import Path +from collections import defaultdict + +BUILD_DIR = "build" +RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results" +GOOGLE_BENCHMARK_COMPARE_SCRIPT = Path("../../../benchmarks/benchmark/tools/compare.py") + + +def extract_algo_name(filename: str) -> str: + # Removes timestamp and UUID prefix + # Matches: 2025-04-17_13-23-38_abcdef_algo.json + return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename) + + +def main() -> None: + print("Comparing most recent benchmarks per algorithm...") + + if not RESULTS_DIR.is_dir(): + print(f"Results directory '{RESULTS_DIR}' not found.") + return + + algo_files = defaultdict(list) + for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True): + algo = extract_algo_name(file.name) + algo_files[algo].append(file) + + for algo, files in algo_files.items(): + if len(files) < 2: + print(f"Skipping {algo} (need at least 2 versions)") + continue + + new, old = files[:2] + name = algo.replace(".json", "").replace(".cpp", "") + + print("────────────────────────────────────────────────────────────") + print(f"Algorithm: {name}") + print("Comparing most recent two runs:") + print(f" OLD: {old.name}") + print(f" NEW: {new.name}\n") + + try: + subprocess.run( + ["python", str(GOOGLE_BENCHMARK_COMPARE_SCRIPT), "benchmarks", old.name, new.name], + cwd=RESULTS_DIR, + check=False, + ) + except Exception as e: + print(f"Error running compare script: {e}") + print() + + +if __name__ == "__main__": + main() diff --git a/tools/scripts/output_benchmarks.py b/tools/scripts/output_benchmarks.py new file mode 100644 index 0000000..30bc4ed --- /dev/null +++ b/tools/scripts/output_benchmarks.py @@ -0,0 +1,61 @@ +# Copyright (c) Brandon Pacewic +# SPDX-License-Identifier: MIT + +import os +import re +import json +from pathlib import Path +from collections import defaultdict + +BUILD_DIR = "build" +RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results" + + +def extract_algo_name(filename: str) -> str: + # Removes timestamp and UUID prefix + # Matches: 2025-04-17_13-23-38_abcdef_algo.json + return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename) + + +def print_benchmark_data(file_path: Path) -> None: + try: + with open(file_path, 'r') as f: + data = json.load(f) + + for bench in data.get("benchmarks", []): + name = bench.get("name", "") + real_time = bench.get("real_time", 0.0) + cpu_time = bench.get("cpu_time", 0.0) + iterations = bench.get("iterations", 0) + print(f"{name:<30} Time: {real_time:>10.2f} CPU: {cpu_time:>10.2f} Iter: {iterations}") + except Exception as e: + print(f"Error reading {file_path.name}: {e}") + + +def main() -> None: + print("Showing most recent benchmark outputs per algorithm...") + + if not RESULTS_DIR.is_dir(): + print(f"Results directory '{RESULTS_DIR}' not found.") + return + + algo_to_files = defaultdict(list) + + for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True): + algo_name = extract_algo_name(file.name) + algo_to_files[algo_name].append(file) + + for algo, files in algo_to_files.items(): + latest = files[0] + display_name = algo.replace(".json", "").replace(".cpp", "") + + print("────────────────────────────────────────────────────────────") + print(f"Algorithm: {display_name}") + print(f"File: {latest.name}\n") + + print_benchmark_data(latest) + print() + + +if __name__ == "__main__": + main() From e87e7886fbb34a2ae5671c8ca0dbf4b7e6a972d3 Mon Sep 17 00:00:00 2001 From: BrandonPacewic <92102436+BrandonPacewic@users.noreply.github.com> Date: Fri, 18 Apr 2025 11:30:42 -0700 Subject: [PATCH 2/3] fix(makefile): spelling error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8164cc4..3dce871 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ help: @echo " bench-out - output the most recent benchmark run" @echo " tools - build project-defined tools (BUILD_TOOLS=ON)" @echo " format - run clang-format on all source files" - @echo " clean - clean build artefacts" + @echo " clean - clean build artifacts" @echo " distclean - clean entire build dir" @echo "" @echo "Knobs (override with VAR=value):" From 01c2e617867f2efbd4881e4aa5f4f111acaaea12 Mon Sep 17 00:00:00 2001 From: BrandonPacewic <92102436+BrandonPacewic@users.noreply.github.com> Date: Fri, 18 Apr 2025 11:32:24 -0700 Subject: [PATCH 3/3] fix(scripts): remove extra whitespace for consistency --- tools/scripts/compare_benchmarks.py | 1 - tools/scripts/output_benchmarks.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/tools/scripts/compare_benchmarks.py b/tools/scripts/compare_benchmarks.py index 31e611c..eeccb74 100644 --- a/tools/scripts/compare_benchmarks.py +++ b/tools/scripts/compare_benchmarks.py @@ -20,7 +20,6 @@ def extract_algo_name(filename: str) -> str: def main() -> None: print("Comparing most recent benchmarks per algorithm...") - if not RESULTS_DIR.is_dir(): print(f"Results directory '{RESULTS_DIR}' not found.") return diff --git a/tools/scripts/output_benchmarks.py b/tools/scripts/output_benchmarks.py index 30bc4ed..181c7ae 100644 --- a/tools/scripts/output_benchmarks.py +++ b/tools/scripts/output_benchmarks.py @@ -34,13 +34,11 @@ def print_benchmark_data(file_path: Path) -> None: def main() -> None: print("Showing most recent benchmark outputs per algorithm...") - if not RESULTS_DIR.is_dir(): print(f"Results directory '{RESULTS_DIR}' not found.") return algo_to_files = defaultdict(list) - for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True): algo_name = extract_algo_name(file.name) algo_to_files[algo_name].append(file)