diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eac630bc..1ad2dd1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,33 @@ jobs: working-directory: example run: cargo r --release --manifest-path ../go-runner/Cargo.toml -- test -bench=. ${{ matrix.target }} + go-runner-benchmarks: + runs-on: codspeed-macro + steps: + - uses: actions/checkout@v4 + with: + lfs: true + submodules: true + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + + - uses: actions/setup-go@v5 + with: + go-version: '1.25' + + - name: Build the benchmark target(s) + run: cargo codspeed build -m walltime + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: walltime + run: cargo codspeed run + check: runs-on: ubuntu-latest if: always() @@ -94,6 +121,7 @@ jobs: - tests - verify-fork-scripts - compat-integration-test-walltime + - go-runner-benchmarks steps: - uses: re-actors/alls-green@release/v1 with: diff --git a/Cargo.lock b/Cargo.lock index 58ecd946..fa661c83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,12 +97,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.2.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" version = "4.5.46" @@ -123,6 +139,7 @@ dependencies = [ "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] @@ -143,12 +160,74 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "codspeed" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b847e05a34be5c38f3f2a5052178a3bd32e6b5702f3ea775efde95c483a539" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.2.16", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-divan-compat" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f0e9fe5eaa39995ec35e46407f7154346cc25bd1300c64c21636f3d00cb2cc" +dependencies = [ + "clap", + "codspeed", + "codspeed-divan-compat-macros", + "codspeed-divan-compat-walltime", + "regex", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88c8babf2a40fd2206a2e030cf020d0d58144cd56e1dc408bfba02cdefb08b4f" +dependencies = [ + "divan-macros", + "itertools", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "codspeed-divan-compat-walltime" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f26092328e12a36704ffc552f379c6405dd94d3149970b79b22d371717c2aae" +dependencies = [ + "cfg-if", + "clap", + "codspeed", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + [[package]] name = "codspeed-go-runner" version = "0.4.2" dependencies = [ "anyhow", "clap", + "codspeed-divan-compat", + "dircpy", "env_logger", "glob", "gosyn", @@ -156,6 +235,7 @@ dependencies = [ "insta", "itertools", "log", + "rand", "regex", "rstest", "semver", @@ -173,6 +253,22 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "console" version = "0.15.11" @@ -194,6 +290,62 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.6" @@ -280,6 +432,28 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dircpy" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88521b0517f5f9d51d11925d8ab4523497dcf947073fa3231a311b63941131c" +dependencies = [ + "jwalk", + "log", + "walkdir", +] + +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -337,6 +511,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + [[package]] name = "fnv" version = "1.0.7" @@ -396,6 +576,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.3.3" @@ -405,7 +596,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -535,6 +726,16 @@ dependencies = [ "syn", ] +[[package]] +name = "jwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56" +dependencies = [ + "crossbeam", + "rayon", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -580,6 +781,18 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nu-ansi-term" version = "0.50.1" @@ -696,6 +909,15 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro-crate" version = "3.3.0" @@ -729,11 +951,60 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" -version = "1.11.2" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -743,15 +1014,21 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" + [[package]] name = "regex-syntax" version = "0.8.6" @@ -827,6 +1104,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.27" @@ -885,6 +1171,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "similar" version = "2.7.0" @@ -953,12 +1245,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.3.3", "once_cell", "rustix", "windows-sys 0.60.2", ] +[[package]] +name = "terminal_size" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + [[package]] name = "test-log" version = "0.2.18" @@ -1173,6 +1475,22 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" @@ -1182,6 +1500,15 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "windows-link" version = "0.1.3" @@ -1361,3 +1688,23 @@ checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags", ] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/go-runner/Cargo.toml b/go-runner/Cargo.toml index 375f981e..42432439 100644 --- a/go-runner/Cargo.toml +++ b/go-runner/Cargo.toml @@ -21,8 +21,15 @@ statrs = { version = "0.18", default-features = false } thiserror = "2.0" tempfile = "3.14" semver = "1.0.27" +dircpy = "0.3.19" [dev-dependencies] +divan = { version = "4.1.0", package = "codspeed-divan-compat" } insta = { version = "1.43", features = ["json", "redactions"] } +rand = "0.9.2" rstest = "0.26" test-log = "0.2.18" + +[[bench]] +name = "go_runner" +harness = false diff --git a/go-runner/benches/go_runner.rs b/go-runner/benches/go_runner.rs new file mode 100644 index 00000000..523940ff --- /dev/null +++ b/go-runner/benches/go_runner.rs @@ -0,0 +1,76 @@ +use codspeed_go_runner::results::raw_result::RawResult; +use tempfile::TempDir; + +#[divan::bench(max_time = std::time::Duration::from_secs(5))] +fn bench_go_runner(bencher: divan::Bencher) { + use std::path::PathBuf; + use tempfile::TempDir; + + let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/projects/example"); + + bencher + .with_inputs(|| { + let temp_dir = TempDir::new().unwrap(); + let profile_dir = temp_dir.path().join("profile"); + let cli = codspeed_go_runner::cli::Cli { + packages: vec!["./...".into()], + dry_run: true, + ..Default::default() + }; + + (profile_dir, cli) + }) + .bench_refs(|(profile_dir, cli)| { + if let Err(error) = + codspeed_go_runner::run_benchmarks(profile_dir, project_dir.as_path(), cli) + { + panic!("Benchmarks couldn't run: {error}"); + } + }); +} + +const TIME_ENTRIES: [usize; 5] = [100_000, 500_000, 1_000_000, 5_000_000, 10_000_000]; + +#[divan::bench(consts = TIME_ENTRIES, max_time = std::time::Duration::from_secs(5))] +fn bench_collect_results(bencher: divan::Bencher) { + fn random_raw_result() -> RawResult { + let times_per_round = (0..N).map(|_| rand::random::() % 1_000_000).collect(); + let iters_per_round = (0..N).map(|_| rand::random::() % 1_000 + 1).collect(); + RawResult { + name: "foo".into(), + uri: "foo".into(), + pid: 42, + codspeed_time_per_round_ns: times_per_round, + codspeed_iters_per_round: iters_per_round, + } + } + + bencher + .with_inputs(|| { + let profile_dir = TempDir::new().unwrap(); + let raw_results = profile_dir.path().join("raw_results"); + std::fs::create_dir(&raw_results).unwrap(); + + for (i, raw_result) in (0..10).map(|_| random_raw_result::()).enumerate() { + let json = serde_json::to_string(&raw_result).unwrap(); + std::fs::write(raw_results.join(format!("{i}.json")), json).unwrap(); + } + + profile_dir + }) + .bench_refs(|profile_dir| { + if let Err(error) = codspeed_go_runner::collect_walltime_results(profile_dir.path()) { + panic!("Collecting results failed: {error}"); + } + + // Ensure that we have a results folder with the pid + let results_dir: std::path::PathBuf = profile_dir.path().join("results"); + assert!(results_dir.exists()); + let result_file = results_dir.join("42.json"); + assert!(result_file.exists()); + }); +} + +fn main() { + divan::main(); +} diff --git a/go-runner/src/builder/discovery.rs b/go-runner/src/builder/discovery.rs index bfaf3433..4b0c0f74 100644 --- a/go-runner/src/builder/discovery.rs +++ b/go-runner/src/builder/discovery.rs @@ -147,6 +147,11 @@ impl GoPackage { let content = std::fs::read_to_string(&file_path) .context(format!("Failed to read test file: {file_path:?}"))?; + // Optimization: Ensure the file contains "func Benchmark" before parsing + if !content.contains("func Benchmark") { + continue; + } + let file = match gosyn::parse_source(&content) { Ok(ast) => ast, Err(e) => { diff --git a/go-runner/src/builder/patcher.rs b/go-runner/src/builder/patcher.rs index 20e2598a..5d0c9e04 100644 --- a/go-runner/src/builder/patcher.rs +++ b/go-runner/src/builder/patcher.rs @@ -46,7 +46,7 @@ pub fn patch_imports>(folder: P) -> anyhow::Result<()> { let content = fs::read_to_string(&go_file).context(format!("Failed to read Go file: {go_file:?}"))?; - let patched_content = patch_imports_for_source(&content)?; + let patched_content = patch_imports_for_source(&content); if patched_content != content { fs::write(&go_file, patched_content) .context(format!("Failed to write patched Go file: {go_file:?}"))?; @@ -61,30 +61,39 @@ pub fn patch_imports>(folder: P) -> anyhow::Result<()> { } /// Internal function to apply import patterns to Go source code -pub fn patch_imports_for_source(source: &str) -> anyhow::Result { - let replace_import = - |mut source: String, import_path: &str, replacement: &str| -> anyhow::Result { - let parsed = gosyn::parse_source(&source)?; - - if let Some(import) = parsed - .imports - .iter() - .find(|import| import.path.value == format!("\"{import_path}\"")) - { - let start_pos = import.path.pos; - let end_pos = start_pos + import.path.value.len(); - - source.replace_range(start_pos..end_pos, replacement); - } - - Ok(source) +pub fn patch_imports_for_source(source: &str) -> String { + let replace_import = |mut source: String, import_path: &str, replacement: &str| -> String { + // Optimization: check if the import path exists in the source before parsing + if !source.contains(import_path) { + return source; + } + + // If we can't parse the source, skip this replacement + // This can happen with template files or malformed Go code + let parsed = match gosyn::parse_source(&source) { + Ok(p) => p, + Err(_) => return source, }; + if let Some(import) = parsed + .imports + .iter() + .find(|import| import.path.value == format!("\"{import_path}\"")) + { + let start_pos = import.path.pos; + let end_pos = start_pos + import.path.value.len(); + + source.replace_range(start_pos..end_pos, replacement); + } + + source + }; + let mut source = replace_import( source.to_string(), "testing", "testing \"github.com/CodSpeedHQ/codspeed-go/testing/testing\"", - )?; + ); // Then replace sub-packages like "testing/synctest" for testing_pkg in &["fstest", "iotest", "quick", "slogtest", "synctest"] { @@ -94,21 +103,19 @@ pub fn patch_imports_for_source(source: &str) -> anyhow::Result { &format!( "{testing_pkg} \"github.com/CodSpeedHQ/codspeed-go/testing/testing/{testing_pkg}\"" ), - )?; + ); } let source = replace_import( source, "github.com/thejerf/slogassert", "\"github.com/CodSpeedHQ/codspeed-go/pkg/slogassert\"", - )?; - let source = replace_import( + ); + replace_import( source, "github.com/frankban/quicktest", "\"github.com/CodSpeedHQ/codspeed-go/pkg/quicktest\"", - )?; - - Ok(source) + ) } /// Patches imports and package in specific test files @@ -393,7 +400,7 @@ func TestExample(t *testing.T) { )] #[case("package_main", PACKAGE_MAIN)] fn test_patch_go_source(#[case] test_name: &str, #[case] source: &str) { - let result = patch_imports_for_source(source).unwrap(); + let result = patch_imports_for_source(source); let result = patch_package_for_source(result).unwrap(); assert_snapshot!(test_name, result); } diff --git a/go-runner/src/builder/templater.rs b/go-runner/src/builder/templater.rs index 67ff5dd6..83d3422b 100644 --- a/go-runner/src/builder/templater.rs +++ b/go-runner/src/builder/templater.rs @@ -21,14 +21,26 @@ struct TemplateData { module_name: String, } -pub fn run>( - package: &BenchmarkPackage, - profile_dir: P, -) -> anyhow::Result<(TempDir, PathBuf)> { - // 1. Copy the whole module to a build directory - let target_dir = TempDir::new()?; - std::fs::create_dir_all(&target_dir).context("Failed to create target directory")?; - utils::copy_dir_recursively(&package.module.dir, &target_dir)?; +/// Runs the templater which sets up a temporary Go project with patched test files and a custom runner. +/// +/// # Returns +/// +/// The path to the generated runner.go file. This should be passed to the `build_binary` function to build +/// the binary that will execute the benchmarks. +pub fn run>(package: &BenchmarkPackage, profile_dir: P) -> anyhow::Result { + // Create a temporary target directory for building the modified Go project. + // NOTE: We don't want to spend time cleanup any temporary files since the code is only + // run on CI servers which clean up themselves. + let target_dir = TempDir::new()?.keep(); + + // 1. Copy the whole git repository to a build directory + let git_root = if let Ok(git_dir) = utils::get_parent_git_repo_path(&package.module.dir) { + git_dir + } else { + warn!("Could not find git repository root. Falling back to module directory as root"); + PathBuf::from(&package.module.dir) + }; + utils::copy_dir_recursively(&git_root, &target_dir)?; // Create a new go-runner.metadata file in the root of the project // @@ -48,7 +60,7 @@ pub fn run>( relative_package_path, }; fs::write( - target_dir.path().join("go-runner.metadata"), + target_dir.join("go-runner.metadata"), serde_json::to_string_pretty(&metadata)?, ) .context("Failed to write go-runner.metadata file")?; @@ -58,19 +70,18 @@ pub fn run>( .test_files() .with_context(|| anyhow::anyhow!("No test files found for package: {}", package.name))?; - // Calculate the relative path from module root to package directory + // Calculate the relative path from git root to package directory let package_dir = Path::new(&package.dir); - let module_dir = Path::new(&package.module.dir); - let relative_package_path = package_dir.strip_prefix(module_dir).context(format!( - "Package dir {:?} is not within module dir {:?}", - package.dir, package.module.dir + let relative_package_path = package_dir.strip_prefix(&git_root).context(format!( + "Package dir {:?} is not within git root {:?}", + package.dir, git_root ))?; debug!("Relative package path: {relative_package_path:?}"); // 2. Patch the imports and package of the test files // - Renames package declarations (to support main package tests and external tests) // - Fixes imports to use our compat packages (e.g., testing/quicktest/testify) - let package_path = target_dir.path().join(relative_package_path); + let package_path = target_dir.join(relative_package_path); let test_file_paths: Vec = files.iter().map(|f| package_path.join(f)).collect(); // If we have external tests (e.g. "package {pkg}_test") they have to be @@ -87,14 +98,19 @@ pub fn run>( } patcher::patch_imports(&target_dir)?; - // 3. Install codspeed-go dependency at the module level (once for the whole module) - patcher::install_codspeed_dependency(&target_dir)?; - - // 3. Handle test files differently based on whether they're external or internal tests - let codspeed_dir = target_dir - .path() - .join(relative_package_path) - .join("codspeed"); + // 3. Install codspeed-go dependency at the package module level + // Find the module directory by getting the relative path from git root + let module_dir = Path::new(&package.module.dir) + .strip_prefix(&git_root) + .map(|relative_module_path| target_dir.join(relative_module_path)) + .unwrap_or_else(|_| { + // Fall back to target_dir if we can't calculate relative path + target_dir.to_path_buf() + }); + patcher::install_codspeed_dependency(&module_dir)?; + + // 4. Handle test files differently based on whether they're external or internal tests + let codspeed_dir = target_dir.join(relative_package_path).join("codspeed"); fs::create_dir_all(&codspeed_dir).context("Failed to create codspeed directory")?; if package.is_external_test_package() { @@ -103,7 +119,7 @@ pub fn run>( // They're now package main and will be built from the subdirectory debug!("Handling external test package - moving files to codspeed/ subdirectory"); for file in files { - let src_path = target_dir.path().join(relative_package_path).join(file); + let src_path = target_dir.join(relative_package_path).join(file); // Rename _test.go to _codspeed.go so it's not treated as a test file let dst_filename = file.replace("_test.go", "_codspeed.go"); let dst_path = codspeed_dir.join(&dst_filename); @@ -116,7 +132,7 @@ pub fn run>( // For internal test packages: rename _test.go to _codspeed.go in place debug!("Handling internal test package - renaming files in place"); for file in files { - let old_path = target_dir.path().join(relative_package_path).join(file); + let old_path = target_dir.join(relative_package_path).join(file); let new_path = old_path.with_file_name( old_path .file_name() @@ -130,7 +146,7 @@ pub fn run>( } } - // 4. Generate the codspeed/runner.go file using the template + // 5. Generate the codspeed/runner.go file using the template let mut handlebars = Handlebars::new(); let template_content = include_str!("template.go"); handlebars.register_template_string("main", template_content)?; @@ -146,5 +162,5 @@ pub fn run>( let runner_path = codspeed_dir.join("runner.go"); fs::write(&runner_path, rendered).context("Failed to write runner.go file")?; - Ok((target_dir, runner_path)) + Ok(runner_path) } diff --git a/go-runner/src/cli.rs b/go-runner/src/cli.rs index 8bc2e2de..607a5936 100644 --- a/go-runner/src/cli.rs +++ b/go-runner/src/cli.rs @@ -16,6 +16,9 @@ pub struct Cli { /// Package patterns to run benchmarks for pub packages: Vec, + + /// Build benchmarks but don't execute them + pub dry_run: bool, } impl Default for Cli { @@ -24,6 +27,7 @@ impl Default for Cli { bench: ".".into(), benchtime: "3s".into(), packages: vec!["./...".into()], + dry_run: false, } } } @@ -39,6 +43,15 @@ impl Cli { } } + /// Parses the command-line arguments into a Cli instance. + /// + /// # Why not use clap or structopt? + /// + /// Unfortunately `go test` supports arguments with a different syntax than those crates. For + /// example, `-bench` and `-benchtime` use single dashes instead of double dashes. + /// + /// We can't do this with clap/structopt, because they only support single dashes for single-letter + /// flags (e.g., `-h`), and double dashes for multi-letter flags (e.g., `--help`). fn parse_args(mut args: impl Iterator) -> Result { let mut instance = Self::default(); @@ -62,11 +75,12 @@ USAGE: OPTIONS: -bench Run only benchmarks matching regexp (defaults to '.') -benchtime Run each benchmark for duration d (defaults to '3s') + --dry-run Build benchmarks but don't execute them -h, --help Print help information -V, --version Print version information SUPPORTED FLAGS: - -bench, -benchtime + -bench, -benchtime, --dry-run UNSUPPORTED FLAGS (will be warned about): -benchmem, -count, -cpu, -cpuprofile, -memprofile, -trace, etc." @@ -95,6 +109,9 @@ UNSUPPORTED FLAGS (will be warned about): s if s.starts_with("-benchtime=") => { instance.benchtime = s.split_once('=').unwrap().1.to_string(); } + "--dry-run" => { + instance.dry_run = true; + } s if s.starts_with('-') => { eprintln!( "warning: flag '{s}' is not supported by CodSpeed Go runner, ignoring" @@ -205,4 +222,32 @@ mod tests { let result = str_to_iter("go-runner test -unknown"); assert!(result.is_ok()); } + + #[test] + fn test_cli_parse_dry_run_flag() { + let cli = str_to_iter("go-runner test --dry-run").unwrap(); + assert!(cli.dry_run); + + let cli = str_to_iter("go-runner test").unwrap(); + assert!(!cli.dry_run); + } + + #[test] + fn test_cli_parse_dry_run_with_other_flags() { + let cli = + str_to_iter("go-runner test --dry-run -bench=BenchmarkFoo -benchtime 5s").unwrap(); + assert!(cli.dry_run); + assert_eq!(cli.bench, "BenchmarkFoo"); + assert_eq!(cli.benchtime, "5s"); + } + + #[test] + fn test_cli_parse_dry_run_with_packages() { + let cli = str_to_iter("go-runner test --dry-run ./pkg1 ./pkg2").unwrap(); + assert!(cli.dry_run); + assert_eq!( + cli.packages, + vec!["./pkg1".to_string(), "./pkg2".to_string()] + ); + } } diff --git a/go-runner/src/integration_tests.rs b/go-runner/src/integration_tests.rs index c82b695a..d6e22e4d 100644 --- a/go-runner/src/integration_tests.rs +++ b/go-runner/src/integration_tests.rs @@ -81,6 +81,7 @@ fn assert_results_snapshots(profile_dir: &Path, project_name: &str) { #[case::example_with_dot_go_folder("example-with-dot-go-folder")] #[case::example_with_vendor("example-with-vendor")] #[case::example_with_test_package("example-with-test-package")] +#[case::example_with_replace("example-with-replace")] #[test_log::test] fn test_build_and_run(#[case] project_name: &str) { let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/go-runner/src/lib.rs b/go-runner/src/lib.rs index c973d091..9840ef36 100644 --- a/go-runner/src/lib.rs +++ b/go-runner/src/lib.rs @@ -4,7 +4,7 @@ use std::{collections::HashMap, path::Path}; pub mod builder; pub mod cli; pub mod prelude; -mod results; +pub mod results; pub mod runner; pub(crate) mod utils; @@ -35,7 +35,7 @@ pub fn run_benchmarks>( // 2. Generate codspeed runners, build binaries, and execute them for package in &packages { info!("Generating custom runner for package: {}", package.name); - let (_target_dir, runner_path) = builder::templater::run(package, &profile_dir)?; + let runner_path = builder::templater::run(package, &profile_dir)?; info!("Building binary for package: {}", package.name); @@ -51,23 +51,29 @@ pub fn run_benchmarks>( } }; - if let Err(error) = runner::run( - &binary_path, - &["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime], - ) { - error!("Failed to run benchmarks for {}: {error}", package.name); - continue; + if !cli.dry_run { + if let Err(error) = runner::run( + &binary_path, + &["-test.bench", &cli.bench, "-test.benchtime", &cli.benchtime], + ) { + error!("Failed to run benchmarks for {}: {error}", package.name); + continue; + } + } else { + info!("Skipping benchmark execution (dry-run mode)"); } } // 3. Collect the results - collect_walltime_results(profile_dir.as_ref())?; + if !cli.dry_run { + collect_walltime_results(profile_dir.as_ref())?; + } Ok(()) } // TODO: This should be merged with codspeed-rust/codspeed/walltime_results.rs -fn collect_walltime_results(profile_dir: &Path) -> anyhow::Result<()> { +pub fn collect_walltime_results(profile_dir: &Path) -> anyhow::Result<()> { let raw_results = results::raw_result::RawResult::parse_folder(profile_dir)?; info!("Parsed {} raw results", raw_results.len()); diff --git a/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example-with-replace.snap b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example-with-replace.snap new file mode 100644 index 00000000..89ed3232 --- /dev/null +++ b/go-runner/src/snapshots/codspeed_go_runner__integration_tests__assert_results_snapshots@example-with-replace.snap @@ -0,0 +1,29 @@ +--- +source: go-runner/src/integration_tests.rs +expression: results +--- +[ + { + "creator": { + "name": "codspeed-go", + "version": "[version]", + "pid": "[pid]" + }, + "instrument": { + "type": "walltime" + }, + "benchmarks": [ + { + "name": "BenchmarkGetValue", + "uri": "go-runner/testdata/projects/example-with-replace/main_test.go::BenchmarkGetValue", + "config": { + "warmup_time_ns": null, + "min_round_time_ns": null, + "max_time_ns": null, + "max_rounds": null + }, + "stats": "[stats]" + } + ] + } +] diff --git a/go-runner/src/utils.rs b/go-runner/src/utils.rs index 091bbc34..43e80b1c 100644 --- a/go-runner/src/utils.rs +++ b/go-runner/src/utils.rs @@ -1,27 +1,16 @@ #[cfg(test)] use crate::prelude::*; +use std::io; use std::path::{Path, PathBuf}; -use std::{fs, io}; pub fn copy_dir_recursively(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { - fs::create_dir_all(&dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; - if ty.is_dir() { - if entry.file_name() == ".git" { - continue; - } - - copy_dir_recursively(entry.path(), dst.as_ref().join(entry.file_name()))?; - } else { - fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; - } - } + let excludes = vec!["node_modules".into(), "target".into()]; + let includes = vec![]; + dircpy::copy_dir_advanced(src, dst, true, true, true, excludes, includes)?; Ok(()) } -fn get_parent_git_repo_path(abs_path: &Path) -> io::Result { +pub fn get_parent_git_repo_path(abs_path: &Path) -> io::Result { if abs_path.join(".git").exists() { Ok(abs_path.to_path_buf()) } else { diff --git a/go-runner/testdata/projects/example-with-replace/dep/dep.go b/go-runner/testdata/projects/example-with-replace/dep/dep.go new file mode 100644 index 00000000..7881d1f2 --- /dev/null +++ b/go-runner/testdata/projects/example-with-replace/dep/dep.go @@ -0,0 +1,5 @@ +package dep + +func GetValue() int { + return 42 +} diff --git a/go-runner/testdata/projects/example-with-replace/dep/go.mod b/go-runner/testdata/projects/example-with-replace/dep/go.mod new file mode 100644 index 00000000..08339d9c --- /dev/null +++ b/go-runner/testdata/projects/example-with-replace/dep/go.mod @@ -0,0 +1,3 @@ +module example-with-replace/dep + +go 1.24.3 diff --git a/go-runner/testdata/projects/example-with-replace/go.mod b/go-runner/testdata/projects/example-with-replace/go.mod new file mode 100644 index 00000000..4ea7da67 --- /dev/null +++ b/go-runner/testdata/projects/example-with-replace/go.mod @@ -0,0 +1,7 @@ +module example-with-replace + +go 1.24.3 + +require example-with-replace/dep v0.0.0 + +replace example-with-replace/dep => ./dep diff --git a/go-runner/testdata/projects/example-with-replace/main.go b/go-runner/testdata/projects/example-with-replace/main.go new file mode 100644 index 00000000..c51dc741 --- /dev/null +++ b/go-runner/testdata/projects/example-with-replace/main.go @@ -0,0 +1,7 @@ +package main + +import "example-with-replace/dep" + +func GetValue() int { + return dep.GetValue() +} diff --git a/go-runner/testdata/projects/example-with-replace/main_test.go b/go-runner/testdata/projects/example-with-replace/main_test.go new file mode 100644 index 00000000..bd5187e2 --- /dev/null +++ b/go-runner/testdata/projects/example-with-replace/main_test.go @@ -0,0 +1,11 @@ +package main + +import ( + "testing" +) + +func BenchmarkGetValue(b *testing.B) { + for i := 0; i < b.N; i++ { + GetValue() + } +} diff --git a/go-runner/tests/pkg_arg.rs b/go-runner/tests/pkg_arg.rs index 0f231e59..ad093f29 100644 --- a/go-runner/tests/pkg_arg.rs +++ b/go-runner/tests/pkg_arg.rs @@ -9,6 +9,7 @@ pub fn test_pkg_arg_filters_correctly() { bench: "BenchmarkBar1".to_string(), benchtime: "1x".to_string(), packages: vec!["./bar".to_string()], + dry_run: false, }; let stdout = run_with_cli("tests/pkg_arg.in", &cli).unwrap(); @@ -27,6 +28,7 @@ pub fn test_pkg_arg_all_packages() { bench: ".".to_string(), benchtime: "1x".to_string(), packages: vec!["./...".to_string()], + dry_run: false, }; let stdout = run_with_cli_multi("tests/pkg_arg.in", &cli).unwrap(); @@ -43,6 +45,7 @@ pub fn test_pkg_arg_multiple_packages() { bench: ".".to_string(), benchtime: "1x".to_string(), packages: vec!["./foo".to_string(), "./bar".to_string()], + dry_run: false, }; let stdout = run_with_cli_multi("tests/pkg_arg.in", &cli).unwrap(); diff --git a/go-runner/tests/utils.rs b/go-runner/tests/utils.rs index 4794b96b..3b1703b3 100644 --- a/go-runner/tests/utils.rs +++ b/go-runner/tests/utils.rs @@ -4,7 +4,7 @@ use std::path::Path; /// Helper function to run a single package with arguments pub fn run_package_with_args(package: &BenchmarkPackage, args: &[&str]) -> anyhow::Result { let profile_dir = tempfile::TempDir::new()?; - let (_dir, runner_path) = builder::templater::run(package, profile_dir.as_ref())?; + let runner_path = builder::templater::run(package, profile_dir.as_ref())?; let binary_path = builder::build_binary(&runner_path)?; runner::run_with_stdout(&binary_path, args) }