feat: add codspeed benchmarks for valgrind

.gitattributes

@@ -0,0 +1 @@
+bench/testdata/* filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -22,6 +22,17 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # Skip installing package docs to avoid wasting time when installing valgrind
+      # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
+      - name: Skip installing package docs
+        if: runner.os == 'Linux'
+        run: |
+          sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
+          path-exclude /usr/share/doc/*
+          path-exclude /usr/share/man/*
+          path-exclude /usr/share/info/*
+          EOF
+
       - name: Update apt-get cache
         run: sudo apt-get update
 

.github/workflows/codspeed.yml

@@ -0,0 +1,100 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: codspeed-macro
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        # IMPORTANT: The binary has to match the architecture of the runner!
+        cmd:
+          - testdata/take_strings-aarch64 varbinview_non_null
+          - echo Hello, World!
+          - ls bench.py
+          - python3 testdata/test.py
+          - stress-ng --cpu 1 --timeout 1s
+          - stress-ng --cpu 4 --timeout 1s
+        valgrind:
+          - "3.26.0"
+          - "3.25.1"
+          - "local"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+      - uses: extractions/setup-just@v3
+
+      # Skip installing package docs to avoid wasting time when installing build dependencies
+      # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
+      - name: Skip installing package docs
+        if: runner.os == 'Linux'
+        run: |
+          sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
+          path-exclude /usr/share/doc/*
+          path-exclude /usr/share/man/*
+          path-exclude /usr/share/info/*
+          EOF
+
+      - name: Cache Valgrind build
+        uses: actions/cache@v4
+        id: valgrind-cache
+        with:
+          path: /tmp/valgrind-build
+          key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-${{ matrix.valgrind == 'local' && hashFiles('coregrind/**', 'include/**', 'VEX/**', 'cachegrind/**', 'callgrind/**', 'dhat/**', 'drd/**', 'helgrind/**', 'lackey/**', 'massif/**', 'memcheck/**', 'none/**', 'exp-bbv/**', 'auxprogs/**', '*.ac', '*.am', '*.in', 'autogen.sh', 'configure*') || 'build' }}
+
+      # Build and install Valgrind
+      - name: Update apt-get cache
+        if: steps.valgrind-cache.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+
+          # Remove existing Valgrind installation
+          sudo apt-get remove -y valgrind || true
+
+      - name: Install build dependencies
+        if: steps.valgrind-cache.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get install -y \
+            build-essential \
+            automake \
+            autoconf \
+            gdb \
+            docbook \
+            docbook-xsl \
+            docbook-xml \
+            xsltproc
+
+      - name: Build Valgrind (${{ matrix.valgrind }})
+        if: steps.valgrind-cache.outputs.cache-hit != 'true'
+        run: just build ${{ matrix.valgrind }}
+
+      - name: Install Valgrind (${{ matrix.valgrind }})
+        run: |
+          just install ${{ matrix.valgrind }}
+
+          # Ensure libc6-dev is installed for Valgrind to work properly
+          sudo apt-get update
+          sudo apt-get install -y libc6-dev stress-ng
+
+      - name: Verify Valgrind build
+        run: /usr/local/bin/valgrind --version
+
+      # Setup benchmarks and run them
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@main
+        env:
+          CODSPEED_PERF_ENABLED: false
+        with:
+          working-directory: bench
+          mode: walltime
+          run: ./bench.py --cmd "${{ matrix.cmd }}" --valgrind-path /usr/local/bin/valgrind

Justfile

@@ -0,0 +1,46 @@
+# Builds a specific valgrind version
+# Usage:
+# - just build 3.24.0: Downloads the specified version from sourceware.org, builds and installs it
+# - just build local: Builds the local source tree
+build version:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    mkdir -p /tmp/valgrind-build
+    rm -rf /tmp/valgrind-build/valgrind-{{ version }}*
+
+    if [ "{{ version }}" = "local" ]; then
+        cp -r . /tmp/valgrind-build/valgrind-local
+    else
+        wget -q -O /tmp/valgrind-build/valgrind-{{ version }}.tar.bz2 \
+            https://sourceware.org/pub/valgrind/valgrind-{{ version }}.tar.bz2
+        tar -xjf /tmp/valgrind-build/valgrind-{{ version }}.tar.bz2 \
+            -C /tmp/valgrind-build
+    fi
+
+    just build-in "/tmp/valgrind-build/valgrind-{{ version }}"
+
+build-in dir:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    cd "{{ dir }}"
+
+    # Check if we need to run autogen.sh (for git checkouts)
+    if [ -f "autogen.sh" ] && [ ! -f "configure" ]; then
+        ./autogen.sh
+    fi
+
+    ./configure
+    make include/vgversion.h
+    make -j$(nproc) -C VEX
+    make -j$(nproc) -C coregrind
+    make -j$(nproc) -C callgrind
+
+
+install version:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    cd "/tmp/valgrind-build/valgrind-{{ version }}"
+    sudo make install
+

bench/bench.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "pytest>=8.4.2",
+#     "pytest-codspeed>=4.2.0",
+# ]
+# ///
+
+import argparse
+import shlex
+import subprocess
+
+import pytest
+
+
+class ValgrindRunner:
+    """Run Valgrind with different configurations."""
+
+    def __init__(
+        self,
+        cmd: str,
+        valgrind_path: str = "valgrind",
+    ):
+        """Initialize valgrind runner.
+
+        Args:
+            cmd: Command to profile (can be a path or arbitrary shell command)
+            valgrind_path: Path to valgrind executable
+        """
+        self.cmd = cmd
+        self.valgrind_path = valgrind_path
+
+        # Verify valgrind is available
+        result = subprocess.run(
+            [self.valgrind_path, "--version"],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}")
+        self.valgrind_version = result.stdout.strip()
+
+    def run_valgrind(self, *args: str) -> None:
+        """Execute valgrind with given arguments.
+
+        Args:
+            *args: Valgrind arguments
+        """
+
+        cmd = [
+            self.valgrind_path,
+            "--tool=callgrind",
+            "--log-file=/dev/null",
+            *args,
+            *shlex.split(self.cmd),
+        ]
+
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(
+                f"Valgrind execution failed with code {result.returncode}\n"
+                f"Stdout:\n{result.stdout}\n"
+                f"Stderr:\n{result.stderr}"
+            )
+
+
+@pytest.fixture
+def runner(request):
+    """Fixture to provide runner instance to tests."""
+    return request.config._valgrind_runner
+
+
+def pytest_generate_tests(metafunc):
+    """Parametrize tests with valgrind configurations."""
+    if "valgrind_args" in metafunc.fixturenames:
+        runner = getattr(metafunc.config, "_valgrind_runner", None)
+        if not runner:
+            return
+
+        # Define valgrind configurations
+        configs = [
+            (["--read-inline-info=no"], "no-inline"),
+            (["--read-inline-info=yes"], "inline"),
+            (
+                [
+                    "--trace-children=yes",
+                    "--cache-sim=yes",
+                    "--I1=32768,8,64",
+                    "--D1=32768,8,64",
+                    "--LL=8388608,16,64",
+                    "--collect-systime=nsec",
+                    "--compress-strings=no",
+                    "--combine-dumps=yes",
+                    "--dump-line=no",
+                    "--read-inline-info=yes",
+                ],
+                "full-with-inline",
+            ),
+            (
+                [
+                    "--trace-children=yes",
+                    "--cache-sim=yes",
+                    "--I1=32768,8,64",
+                    "--D1=32768,8,64",
+                    "--LL=8388608,16,64",
+                    "--collect-systime=nsec",
+                    "--compress-strings=no",
+                    "--combine-dumps=yes",
+                    "--dump-line=no",
+                ],
+                "full-no-inline",
+            ),
+        ]
+
+        # Create test IDs with format: valgrind-version, command, config-name
+        test_ids = [
+            f"{runner.valgrind_version}, {runner.cmd}, {config_name}"
+            for _, config_name in configs
+        ]
+
+        # Parametrize with just the args
+        metafunc.parametrize(
+            "valgrind_args",
+            [args for args, _ in configs],
+            ids=test_ids,
+        )
+
+
+@pytest.mark.benchmark
+def test_valgrind(runner, valgrind_args):
+    if runner:
+        runner.run_valgrind(*valgrind_args)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Benchmark Valgrind with pytest-codspeed",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with a binary path
+  uv run bench.py --cmd /path/to/binary
+
+  # Run with an arbitrary command
+  uv run bench.py --cmd 'echo "hello world"'
+
+  # Run with custom valgrind installation
+  uv run bench.py --cmd /usr/bin/ls --valgrind-path /usr/local/bin/valgrind
+        """,
+    )
+
+    parser.add_argument(
+        "--cmd",
+        type=str,
+        required=True,
+        help="Command to profile (can be a path to a binary or any arbitrary command)",
+    )
+    parser.add_argument(
+        "--valgrind-path",
+        type=str,
+        default="valgrind",
+        help="Path to valgrind executable (default: valgrind)",
+    )
+    args = parser.parse_args()
+
+    # Create runner instance
+    runner = ValgrindRunner(
+        cmd=args.cmd,
+        valgrind_path=args.valgrind_path,
+    )
+    print(f"Valgrind version: {runner.valgrind_version}")
+    print(f"Command: {args.cmd}")
+
+    # Plugin to pass runner to tests
+    class RunnerPlugin:
+        def pytest_configure(self, config):
+            config._valgrind_runner = runner
+
+    exit_code = pytest.main(
+        [__file__, "-v", "--codspeed", "--codspeed-warmup-time=0", "--codspeed-max-time=5"],
+        plugins=[RunnerPlugin()],
+    )
+    if exit_code != 0 and exit_code != 5:
+        print(f"Benchmark execution returned exit code: {exit_code}")
+
+
+if __name__ == "__main__":
+    main()

bench/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = testdata __pycache__ .pytest_cache *.egg-info