diff --git a/Makefile b/Makefile index ca4cbc7..3dce871 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,12 @@ bench: build run-benchmarks: bench @cmake --build "$(BUILD_DIR)" --target run_all_benchmarks -- -j$(JOBS) +bench-diff: + @python tools/scripts/compare_benchmarks.py + +bench-out: + @python tools/scripts/output_benchmarks.py + tools: BUILD_TOOLS := ON tools: build @@ -54,9 +60,11 @@ help: @echo " test - run ctest (RUN_TESTS=ON)" @echo " bench - build & run benchmarks (BUILD_BENCH=ON)" @echo " run-benchmarks - run all benchmarks" + @echo " bench-diff - compare the two most recent benchmark runs" + @echo " bench-out - output the most recent benchmark run" @echo " tools - build project-defined tools (BUILD_TOOLS=ON)" @echo " format - run clang-format on all source files" - @echo " clean - clean build artefacts" + @echo " clean - clean build artifacts" @echo " distclean - clean entire build dir" @echo "" @echo "Knobs (override with VAR=value):" diff --git a/tools/scripts/compare_benchmarks.py b/tools/scripts/compare_benchmarks.py new file mode 100644 index 0000000..eeccb74 --- /dev/null +++ b/tools/scripts/compare_benchmarks.py @@ -0,0 +1,58 @@ +# Copyright (c) Brandon Pacewic +# SPDX-License-Identifier: MIT + +import os +import re +import subprocess +from pathlib import Path +from collections import defaultdict + +BUILD_DIR = "build" +RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results" +GOOGLE_BENCHMARK_COMPARE_SCRIPT = Path("../../../benchmarks/benchmark/tools/compare.py") + + +def extract_algo_name(filename: str) -> str: + # Removes timestamp and UUID prefix + # Matches: 2025-04-17_13-23-38_abcdef_algo.json + return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename) + + +def main() -> None: + print("Comparing most recent benchmarks per algorithm...") + if not RESULTS_DIR.is_dir(): + print(f"Results directory '{RESULTS_DIR}' not found.") + return + + algo_files = defaultdict(list) + for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True): + algo = extract_algo_name(file.name) + algo_files[algo].append(file) + + for algo, files in algo_files.items(): + if len(files) < 2: + print(f"Skipping {algo} (need at least 2 versions)") + continue + + new, old = files[:2] + name = algo.replace(".json", "").replace(".cpp", "") + + print("────────────────────────────────────────────────────────────") + print(f"Algorithm: {name}") + print("Comparing most recent two runs:") + print(f" OLD: {old.name}") + print(f" NEW: {new.name}\n") + + try: + subprocess.run( + ["python", str(GOOGLE_BENCHMARK_COMPARE_SCRIPT), "benchmarks", old.name, new.name], + cwd=RESULTS_DIR, + check=False, + ) + except Exception as e: + print(f"Error running compare script: {e}") + print() + + +if __name__ == "__main__": + main() diff --git a/tools/scripts/output_benchmarks.py b/tools/scripts/output_benchmarks.py new file mode 100644 index 0000000..181c7ae --- /dev/null +++ b/tools/scripts/output_benchmarks.py @@ -0,0 +1,59 @@ +# Copyright (c) Brandon Pacewic +# SPDX-License-Identifier: MIT + +import os +import re +import json +from pathlib import Path +from collections import defaultdict + +BUILD_DIR = "build" +RESULTS_DIR = Path(BUILD_DIR) / "benchmarks" / "results" + + +def extract_algo_name(filename: str) -> str: + # Removes timestamp and UUID prefix + # Matches: 2025-04-17_13-23-38_abcdef_algo.json + return re.sub(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[^_]+_[^_]+_", "", filename) + + +def print_benchmark_data(file_path: Path) -> None: + try: + with open(file_path, 'r') as f: + data = json.load(f) + + for bench in data.get("benchmarks", []): + name = bench.get("name", "") + real_time = bench.get("real_time", 0.0) + cpu_time = bench.get("cpu_time", 0.0) + iterations = bench.get("iterations", 0) + print(f"{name:<30} Time: {real_time:>10.2f} CPU: {cpu_time:>10.2f} Iter: {iterations}") + except Exception as e: + print(f"Error reading {file_path.name}: {e}") + + +def main() -> None: + print("Showing most recent benchmark outputs per algorithm...") + if not RESULTS_DIR.is_dir(): + print(f"Results directory '{RESULTS_DIR}' not found.") + return + + algo_to_files = defaultdict(list) + for file in sorted(RESULTS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True): + algo_name = extract_algo_name(file.name) + algo_to_files[algo_name].append(file) + + for algo, files in algo_to_files.items(): + latest = files[0] + display_name = algo.replace(".json", "").replace(".cpp", "") + + print("────────────────────────────────────────────────────────────") + print(f"Algorithm: {display_name}") + print(f"File: {latest.name}\n") + + print_benchmark_data(latest) + print() + + +if __name__ == "__main__": + main()