diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..7785f502d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +bench/testdata/* filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8c8f3804..3ae16aa5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,17 @@ jobs: steps: - uses: actions/checkout@v4 + # Skip installing package docs to avoid wasting time when installing valgrind + # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336 + - name: Skip installing package docs + if: runner.os == 'Linux' + run: | + sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF' + path-exclude /usr/share/doc/* + path-exclude /usr/share/man/* + path-exclude /usr/share/info/* + EOF + - name: Update apt-get cache run: sudo apt-get update diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..623a84d20 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,100 @@ +name: CodSpeed Benchmarks + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +jobs: + benchmarks: + runs-on: codspeed-macro + timeout-minutes: 20 + strategy: + matrix: + # IMPORTANT: The binary has to match the architecture of the runner! + cmd: + - testdata/take_strings-aarch64 varbinview_non_null + - echo Hello, World! + - ls bench.py + - python3 testdata/test.py + - stress-ng --cpu 1 --timeout 1s + - stress-ng --cpu 4 --timeout 1s + valgrind: + - "3.26.0" + - "3.25.1" + - "local" + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - uses: extractions/setup-just@v3 + + # Skip installing package docs to avoid wasting time when installing build dependencies + # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336 + - name: Skip installing package docs + if: runner.os == 'Linux' + run: | + sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF' + path-exclude /usr/share/doc/* + path-exclude /usr/share/man/* + path-exclude /usr/share/info/* + EOF + + - name: Cache Valgrind build + uses: actions/cache@v4 + id: valgrind-cache + with: + path: /tmp/valgrind-build + key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-${{ matrix.valgrind == 'local' && hashFiles('coregrind/**', 'include/**', 'VEX/**', 'cachegrind/**', 'callgrind/**', 'dhat/**', 'drd/**', 'helgrind/**', 'lackey/**', 'massif/**', 'memcheck/**', 'none/**', 'exp-bbv/**', 'auxprogs/**', '*.ac', '*.am', '*.in', 'autogen.sh', 'configure*') || 'build' }} + + # Build and install Valgrind + - name: Update apt-get cache + if: steps.valgrind-cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update + + # Remove existing Valgrind installation + sudo apt-get remove -y valgrind || true + + - name: Install build dependencies + if: steps.valgrind-cache.outputs.cache-hit != 'true' + run: | + sudo apt-get install -y \ + build-essential \ + automake \ + autoconf \ + gdb \ + docbook \ + docbook-xsl \ + docbook-xml \ + xsltproc + + - name: Build Valgrind (${{ matrix.valgrind }}) + if: steps.valgrind-cache.outputs.cache-hit != 'true' + run: just build ${{ matrix.valgrind }} + + - name: Install Valgrind (${{ matrix.valgrind }}) + run: | + just install ${{ matrix.valgrind }} + + # Ensure libc6-dev is installed for Valgrind to work properly + sudo apt-get update + sudo apt-get install -y libc6-dev stress-ng + + - name: Verify Valgrind build + run: /usr/local/bin/valgrind --version + + # Setup benchmarks and run them + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Run the benchmarks + uses: CodSpeedHQ/action@main + env: + CODSPEED_PERF_ENABLED: false + with: + working-directory: bench + mode: walltime + run: ./bench.py --cmd "${{ matrix.cmd }}" --valgrind-path /usr/local/bin/valgrind diff --git a/Justfile b/Justfile new file mode 100644 index 000000000..916036cba --- /dev/null +++ b/Justfile @@ -0,0 +1,46 @@ +# Builds a specific valgrind version +# Usage: +# - just build 3.24.0: Downloads the specified version from sourceware.org, builds and installs it +# - just build local: Builds the local source tree +build version: + #!/usr/bin/env bash + set -euo pipefail + + mkdir -p /tmp/valgrind-build + rm -rf /tmp/valgrind-build/valgrind-{{ version }}* + + if [ "{{ version }}" = "local" ]; then + cp -r . /tmp/valgrind-build/valgrind-local + else + wget -q -O /tmp/valgrind-build/valgrind-{{ version }}.tar.bz2 \ + https://sourceware.org/pub/valgrind/valgrind-{{ version }}.tar.bz2 + tar -xjf /tmp/valgrind-build/valgrind-{{ version }}.tar.bz2 \ + -C /tmp/valgrind-build + fi + + just build-in "/tmp/valgrind-build/valgrind-{{ version }}" + +build-in dir: + #!/usr/bin/env bash + set -euo pipefail + cd "{{ dir }}" + + # Check if we need to run autogen.sh (for git checkouts) + if [ -f "autogen.sh" ] && [ ! -f "configure" ]; then + ./autogen.sh + fi + + ./configure + make include/vgversion.h + make -j$(nproc) -C VEX + make -j$(nproc) -C coregrind + make -j$(nproc) -C callgrind + + +install version: + #!/usr/bin/env bash + set -euo pipefail + + cd "/tmp/valgrind-build/valgrind-{{ version }}" + sudo make install + diff --git a/bench/bench.py b/bench/bench.py new file mode 100755 index 000000000..0c54194e7 --- /dev/null +++ b/bench/bench.py @@ -0,0 +1,193 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "pytest>=8.4.2", +# "pytest-codspeed>=4.2.0", +# ] +# /// + +import argparse +import shlex +import subprocess + +import pytest + + +class ValgrindRunner: + """Run Valgrind with different configurations.""" + + def __init__( + self, + cmd: str, + valgrind_path: str = "valgrind", + ): + """Initialize valgrind runner. + + Args: + cmd: Command to profile (can be a path or arbitrary shell command) + valgrind_path: Path to valgrind executable + """ + self.cmd = cmd + self.valgrind_path = valgrind_path + + # Verify valgrind is available + result = subprocess.run( + [self.valgrind_path, "--version"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}") + self.valgrind_version = result.stdout.strip() + + def run_valgrind(self, *args: str) -> None: + """Execute valgrind with given arguments. + + Args: + *args: Valgrind arguments + """ + + cmd = [ + self.valgrind_path, + "--tool=callgrind", + "--log-file=/dev/null", + *args, + *shlex.split(self.cmd), + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"Valgrind execution failed with code {result.returncode}\n" + f"Stdout:\n{result.stdout}\n" + f"Stderr:\n{result.stderr}" + ) + + +@pytest.fixture +def runner(request): + """Fixture to provide runner instance to tests.""" + return request.config._valgrind_runner + + +def pytest_generate_tests(metafunc): + """Parametrize tests with valgrind configurations.""" + if "valgrind_args" in metafunc.fixturenames: + runner = getattr(metafunc.config, "_valgrind_runner", None) + if not runner: + return + + # Define valgrind configurations + configs = [ + (["--read-inline-info=no"], "no-inline"), + (["--read-inline-info=yes"], "inline"), + ( + [ + "--trace-children=yes", + "--cache-sim=yes", + "--I1=32768,8,64", + "--D1=32768,8,64", + "--LL=8388608,16,64", + "--collect-systime=nsec", + "--compress-strings=no", + "--combine-dumps=yes", + "--dump-line=no", + "--read-inline-info=yes", + ], + "full-with-inline", + ), + ( + [ + "--trace-children=yes", + "--cache-sim=yes", + "--I1=32768,8,64", + "--D1=32768,8,64", + "--LL=8388608,16,64", + "--collect-systime=nsec", + "--compress-strings=no", + "--combine-dumps=yes", + "--dump-line=no", + ], + "full-no-inline", + ), + ] + + # Create test IDs with format: valgrind-version, command, config-name + test_ids = [ + f"{runner.valgrind_version}, {runner.cmd}, {config_name}" + for _, config_name in configs + ] + + # Parametrize with just the args + metafunc.parametrize( + "valgrind_args", + [args for args, _ in configs], + ids=test_ids, + ) + + +@pytest.mark.benchmark +def test_valgrind(runner, valgrind_args): + if runner: + runner.run_valgrind(*valgrind_args) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark Valgrind with pytest-codspeed", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run with a binary path + uv run bench.py --cmd /path/to/binary + + # Run with an arbitrary command + uv run bench.py --cmd 'echo "hello world"' + + # Run with custom valgrind installation + uv run bench.py --cmd /usr/bin/ls --valgrind-path /usr/local/bin/valgrind + """, + ) + + parser.add_argument( + "--cmd", + type=str, + required=True, + help="Command to profile (can be a path to a binary or any arbitrary command)", + ) + parser.add_argument( + "--valgrind-path", + type=str, + default="valgrind", + help="Path to valgrind executable (default: valgrind)", + ) + args = parser.parse_args() + + # Create runner instance + runner = ValgrindRunner( + cmd=args.cmd, + valgrind_path=args.valgrind_path, + ) + print(f"Valgrind version: {runner.valgrind_version}") + print(f"Command: {args.cmd}") + + # Plugin to pass runner to tests + class RunnerPlugin: + def pytest_configure(self, config): + config._valgrind_runner = runner + + exit_code = pytest.main( + [__file__, "-v", "--codspeed", "--codspeed-warmup-time=0", "--codspeed-max-time=5"], + plugins=[RunnerPlugin()], + ) + if exit_code != 0 and exit_code != 5: + print(f"Benchmark execution returned exit code: {exit_code}") + + +if __name__ == "__main__": + main() diff --git a/bench/pytest.ini b/bench/pytest.ini new file mode 100644 index 000000000..1afcedaf6 --- /dev/null +++ b/bench/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = testdata __pycache__ .pytest_cache *.egg-info diff --git a/bench/testdata/take_strings-aarch64 b/bench/testdata/take_strings-aarch64 new file mode 100755 index 000000000..078cfa34f --- /dev/null +++ b/bench/testdata/take_strings-aarch64 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d241a1c2932e11d4b5226d193ecf7c120bb881f5f0108884071048dcd5bd6696 +size 282407216 diff --git a/bench/testdata/take_strings-x86_64 b/bench/testdata/take_strings-x86_64 new file mode 100755 index 000000000..d10feaee7 --- /dev/null +++ b/bench/testdata/take_strings-x86_64 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c184f81f7046a8a78cb272ac4a1c7ad616b5e3dd20dcc40638f1db485abc5b22 +size 272199232 diff --git a/bench/testdata/test.py b/bench/testdata/test.py new file mode 100644 index 000000000..c0dfc2f51 --- /dev/null +++ b/bench/testdata/test.py @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf603e7740f7f7cbf211c7b240f8426c0bf602353290cdb3c9a52adbb0dfaec1 +size 22